diff options
Diffstat (limited to 'contrail-analytics')
60 files changed, 7868 insertions, 0 deletions
diff --git a/contrail-analytics/Makefile b/contrail-analytics/Makefile new file mode 100644 index 0000000..378713f --- /dev/null +++ b/contrail-analytics/Makefile @@ -0,0 +1,10 @@ +#!/usr/bin/make +PYTHON := /usr/bin/env python + +bin/charm_helpers_sync.py: + @mkdir -p bin + @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ + > bin/charm_helpers_sync.py + +sync: bin/charm_helpers_sync.py + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-sync.yaml diff --git a/contrail-analytics/README.md b/contrail-analytics/README.md new file mode 100644 index 0000000..ebd111d --- /dev/null +++ b/contrail-analytics/README.md @@ -0,0 +1,45 @@ +Overview +-------- + +OpenContrail (www.opencontrail.org) is a fully featured Software Defined +Networking (SDN) solution for private clouds. It supports high performance +isolated tenant networks without requiring external hardware support. It +provides a Neutron plugin to integrate with OpenStack. + +This charm provides the analytics node component which includes +contrail-collector, contrail-query-engine and contrail-analytics-api services. + +Only OpenStack Mitaka or newer is supported. +Only for Contrail 4.0 for now. +Juju 2.0 is required. + +Usage +----- + +Contrail Controller and Contrail AnalyticsDB are prerequisite services to deploy. +Once ready, deploy and relate as follows: + + juju deploy contrail-analytics + juju add-relation contrail-analytics contrail-analyticsdb + juju add-relation contrail-analytics contrail-controller + +Resource +-------- + +The charm requires docker image with Contrail Analytics as a resource. +It can be provided as usual for Juju 2.0 in deploy command or +through attach-resource: + + juju attach contrail-analytics contrail-analytics="$PATH_TO_IMAGE" + +High Availability (HA) +---------------------- + +Multiple units of this charm can be deployed to support HA deployments: + + juju add-unit contrail-analytics + +Relating to haproxy charm (http-services relation) allows multiple units to be +load balanced: + + juju add-relation contrail-analytics haproxy diff --git a/contrail-analytics/charm-helpers-sync.yaml b/contrail-analytics/charm-helpers-sync.yaml new file mode 100644 index 0000000..933434f --- /dev/null +++ b/contrail-analytics/charm-helpers-sync.yaml @@ -0,0 +1,7 @@ +branch: lp:charm-helpers +destination: hooks/charmhelpers +include: + - core + - fetch + - osplatform + - contrib.network diff --git a/contrail-analytics/config.yaml b/contrail-analytics/config.yaml new file mode 100644 index 0000000..2834bcc --- /dev/null +++ b/contrail-analytics/config.yaml @@ -0,0 +1,7 @@ +options: + control-network: + type: string + description: | + The IP address and netmask of the control network (e.g. 192.168.0.0/24). + This network will be used for Contrail endpoints. + If not specified, default network will be used. diff --git a/contrail-analytics/copyright b/contrail-analytics/copyright new file mode 100644 index 0000000..b48ce83 --- /dev/null +++ b/contrail-analytics/copyright @@ -0,0 +1,17 @@ +Format: http://dep.debian.net/deps/dep5/ + +Files: * +Copyright: Copyright 2016, Canonical Ltd and Juniper Networks Ltd., All Rights Reserved. +License: GPL-3 + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/contrail-analytics/hooks/analytics-cluster-relation-joined b/contrail-analytics/hooks/analytics-cluster-relation-joined new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/analytics-cluster-relation-joined @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/charmhelpers/__init__.py b/contrail-analytics/hooks/charmhelpers/__init__.py new file mode 100644 index 0000000..e7aa471 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/__init__.py @@ -0,0 +1,97 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Bootstrap charm-helpers, installing its dependencies if necessary using +# only standard libraries. +from __future__ import print_function +from __future__ import absolute_import + +import functools +import inspect +import subprocess +import sys + +try: + import six # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) + import six # flake8: noqa + +try: + import yaml # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + import yaml # flake8: noqa + + +# Holds a list of mapping of mangled function names that have been deprecated +# using the @deprecate decorator below. This is so that the warning is only +# printed once for each usage of the function. +__deprecated_functions = {} + + +def deprecate(warning, date=None, log=None): + """Add a deprecation warning the first time the function is used. + The date, which is a string in semi-ISO8660 format indicate the year-month + that the function is officially going to be removed. + + usage: + + @deprecate('use core/fetch/add_source() instead', '2017-04') + def contributed_add_source_thing(...): + ... + + And it then prints to the log ONCE that the function is deprecated. + The reason for passing the logging function (log) is so that hookenv.log + can be used for a charm if needed. + + :param warning: String to indicat where it has moved ot. + :param date: optional sting, in YYYY-MM format to indicate when the + function will definitely (probably) be removed. + :param log: The log function to call to log. If not, logs to stdout + """ + def wrap(f): + + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + try: + module = inspect.getmodule(f) + file = inspect.getsourcefile(f) + lines = inspect.getsourcelines(f) + f_name = "{}-{}-{}..{}-{}".format( + module.__name__, file, lines[0], lines[-1], f.__name__) + except (IOError, TypeError): + # assume it was local, so just use the name of the function + f_name = f.__name__ + if f_name not in __deprecated_functions: + __deprecated_functions[f_name] = True + s = "DEPRECATION WARNING: Function {} is being removed".format( + f.__name__) + if date: + s = "{} on/around {}".format(s, date) + if warning: + s = "{} : {}".format(s, warning) + if log: + log(s) + else: + print(s) + return f(*args, **kwargs) + return wrapped_f + return wrap diff --git a/contrail-analytics/hooks/charmhelpers/contrib/__init__.py b/contrail-analytics/hooks/charmhelpers/contrib/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/contrib/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrail-analytics/hooks/charmhelpers/contrib/network/__init__.py b/contrail-analytics/hooks/charmhelpers/contrib/network/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/contrib/network/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrail-analytics/hooks/charmhelpers/contrib/network/ip.py b/contrail-analytics/hooks/charmhelpers/contrib/network/ip.py new file mode 100644 index 0000000..15f6596 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/contrib/network/ip.py @@ -0,0 +1,593 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import re +import subprocess +import six +import socket + +from functools import partial + +from charmhelpers.fetch import apt_install, apt_update +from charmhelpers.core.hookenv import ( + config, + log, + network_get_primary_address, + unit_get, + WARNING, +) + +from charmhelpers.core.host import ( + lsb_release, + CompareHostReleases, +) + +try: + import netifaces +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) + import netifaces + +try: + import netaddr +except ImportError: + apt_update(fatal=True) + if six.PY2: + apt_install('python-netaddr', fatal=True) + else: + apt_install('python3-netaddr', fatal=True) + import netaddr + + +def _validate_cidr(network): + try: + netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + +def no_ip_found_error_out(network): + errmsg = ("No IP address found in network(s): %s" % network) + raise ValueError(errmsg) + + +def _get_ipv6_network_from_address(address): + """Get an netaddr.IPNetwork for the given IPv6 address + :param address: a dict as returned by netifaces.ifaddresses + :returns netaddr.IPNetwork: None if the address is a link local or loopback + address + """ + if address['addr'].startswith('fe80') or address['addr'] == "::1": + return None + + prefix = address['netmask'].split("/") + if len(prefix) > 1: + netmask = prefix[1] + else: + netmask = address['netmask'] + return netaddr.IPNetwork("%s/%s" % (address['addr'], + netmask)) + + +def get_address_in_network(network, fallback=None, fatal=False): + """Get an IPv4 or IPv6 address within the network from the host. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. Supports multiple networks as a space-delimited list. + :param fallback (str): If no address is found, return fallback. + :param fatal (boolean): If no address is found, fallback is not + set and fatal is True then exit(1). + """ + if network is None: + if fallback is not None: + return fallback + + if fatal: + no_ip_found_error_out(network) + else: + return None + + networks = network.split() or [network] + for network in networks: + _validate_cidr(network) + network = netaddr.IPNetwork(network) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if network.version == 4 and netifaces.AF_INET in addresses: + for addr in addresses[netifaces.AF_INET]: + cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'], + addr['netmask'])) + if cidr in network: + return str(cidr.ip) + + if network.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + cidr = _get_ipv6_network_from_address(addr) + if cidr and cidr in network: + return str(cidr.ip) + + if fallback is not None: + return fallback + + if fatal: + no_ip_found_error_out(network) + + return None + + +def is_ipv6(address): + """Determine whether provided address is IPv6 or not.""" + try: + address = netaddr.IPAddress(address) + except netaddr.AddrFormatError: + # probably a hostname - so not an address at all! + return False + + return address.version == 6 + + +def is_address_in_network(network, address): + """ + Determine whether the provided address is within a network range. + + :param network (str): CIDR presentation format. For example, + '192.168.1.0/24'. + :param address: An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :returns boolean: Flag indicating whether address is in network. + """ + try: + network = netaddr.IPNetwork(network) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Network (%s) is not in CIDR presentation format" % + network) + + try: + address = netaddr.IPAddress(address) + except (netaddr.core.AddrFormatError, ValueError): + raise ValueError("Address (%s) is not in correct presentation format" % + address) + + if address in network: + return True + else: + return False + + +def _get_for_address(address, key): + """Retrieve an attribute of or the physical interface that + the IP address provided could be bound to. + + :param address (str): An individual IPv4 or IPv6 address without a net + mask or subnet prefix. For example, '192.168.1.1'. + :param key: 'iface' for the physical interface name or an attribute + of the configured interface, for example 'netmask'. + :returns str: Requested attribute or None if address is not bindable. + """ + address = netaddr.IPAddress(address) + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + if address.version == 4 and netifaces.AF_INET in addresses: + addr = addresses[netifaces.AF_INET][0]['addr'] + netmask = addresses[netifaces.AF_INET][0]['netmask'] + network = netaddr.IPNetwork("%s/%s" % (addr, netmask)) + cidr = network.cidr + if address in cidr: + if key == 'iface': + return iface + else: + return addresses[netifaces.AF_INET][0][key] + + if address.version == 6 and netifaces.AF_INET6 in addresses: + for addr in addresses[netifaces.AF_INET6]: + network = _get_ipv6_network_from_address(addr) + if not network: + continue + + cidr = network.cidr + if address in cidr: + if key == 'iface': + return iface + elif key == 'netmask' and cidr: + return str(cidr).split('/')[1] + else: + return addr[key] + return None + + +get_iface_for_address = partial(_get_for_address, key='iface') + + +get_netmask_for_address = partial(_get_for_address, key='netmask') + + +def resolve_network_cidr(ip_address): + ''' + Resolves the full address cidr of an ip_address based on + configured network interfaces + ''' + netmask = get_netmask_for_address(ip_address) + return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr) + + +def format_ipv6_addr(address): + """If address is IPv6, wrap it in '[]' otherwise return None. + + This is required by most configuration files when specifying IPv6 + addresses. + """ + if is_ipv6(address): + return "[%s]" % address + + return None + + +def is_ipv6_disabled(): + try: + result = subprocess.check_output( + ['sysctl', 'net.ipv6.conf.all.disable_ipv6'], + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + return True + if six.PY3: + result = result.decode('UTF-8') + return "net.ipv6.conf.all.disable_ipv6 = 1" in result + + +def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, + fatal=True, exc_list=None): + """Return the assigned IP address for a given interface, if any. + + :param iface: network interface on which address(es) are expected to + be found. + :param inet_type: inet address family + :param inc_aliases: include alias interfaces in search + :param fatal: if True, raise exception if address not found + :param exc_list: list of addresses to ignore + :return: list of ip addresses + """ + # Extract nic if passed /dev/ethX + if '/' in iface: + iface = iface.split('/')[-1] + + if not exc_list: + exc_list = [] + + try: + inet_num = getattr(netifaces, inet_type) + except AttributeError: + raise Exception("Unknown inet type '%s'" % str(inet_type)) + + interfaces = netifaces.interfaces() + if inc_aliases: + ifaces = [] + for _iface in interfaces: + if iface == _iface or _iface.split(':')[0] == iface: + ifaces.append(_iface) + + if fatal and not ifaces: + raise Exception("Invalid interface '%s'" % iface) + + ifaces.sort() + else: + if iface not in interfaces: + if fatal: + raise Exception("Interface '%s' not found " % (iface)) + else: + return [] + + else: + ifaces = [iface] + + addresses = [] + for netiface in ifaces: + net_info = netifaces.ifaddresses(netiface) + if inet_num in net_info: + for entry in net_info[inet_num]: + if 'addr' in entry and entry['addr'] not in exc_list: + addresses.append(entry['addr']) + + if fatal and not addresses: + raise Exception("Interface '%s' doesn't have any %s addresses." % + (iface, inet_type)) + + return sorted(addresses) + + +get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET') + + +def get_iface_from_addr(addr): + """Work out on which interface the provided address is configured.""" + for iface in netifaces.interfaces(): + addresses = netifaces.ifaddresses(iface) + for inet_type in addresses: + for _addr in addresses[inet_type]: + _addr = _addr['addr'] + # link local + ll_key = re.compile("(.+)%.*") + raw = re.match(ll_key, _addr) + if raw: + _addr = raw.group(1) + + if _addr == addr: + log("Address '%s' is configured on iface '%s'" % + (addr, iface)) + return iface + + msg = "Unable to infer net iface on which '%s' is configured" % (addr) + raise Exception(msg) + + +def sniff_iface(f): + """Ensure decorated function is called with a value for iface. + + If no iface provided, inject net iface inferred from unit private address. + """ + def iface_sniffer(*args, **kwargs): + if not kwargs.get('iface', None): + kwargs['iface'] = get_iface_from_addr(unit_get('private-address')) + + return f(*args, **kwargs) + + return iface_sniffer + + +@sniff_iface +def get_ipv6_addr(iface=None, inc_aliases=False, fatal=True, exc_list=None, + dynamic_only=True): + """Get assigned IPv6 address for a given interface. + + Returns list of addresses found. If no address found, returns empty list. + + If iface is None, we infer the current primary interface by doing a reverse + lookup on the unit private-address. + + We currently only support scope global IPv6 addresses i.e. non-temporary + addresses. If no global IPv6 address is found, return the first one found + in the ipv6 address list. + + :param iface: network interface on which ipv6 address(es) are expected to + be found. + :param inc_aliases: include alias interfaces in search + :param fatal: if True, raise exception if address not found + :param exc_list: list of addresses to ignore + :param dynamic_only: only recognise dynamic addresses + :return: list of ipv6 addresses + """ + addresses = get_iface_addr(iface=iface, inet_type='AF_INET6', + inc_aliases=inc_aliases, fatal=fatal, + exc_list=exc_list) + + if addresses: + global_addrs = [] + for addr in addresses: + key_scope_link_local = re.compile("^fe80::..(.+)%(.+)") + m = re.match(key_scope_link_local, addr) + if m: + eui_64_mac = m.group(1) + iface = m.group(2) + else: + global_addrs.append(addr) + + if global_addrs: + # Make sure any found global addresses are not temporary + cmd = ['ip', 'addr', 'show', iface] + out = subprocess.check_output(cmd).decode('UTF-8') + if dynamic_only: + key = re.compile("inet6 (.+)/[0-9]+ scope global.* dynamic.*") + else: + key = re.compile("inet6 (.+)/[0-9]+ scope global.*") + + addrs = [] + for line in out.split('\n'): + line = line.strip() + m = re.match(key, line) + if m and 'temporary' not in line: + # Return the first valid address we find + for addr in global_addrs: + if m.group(1) == addr: + if not dynamic_only or \ + m.group(1).endswith(eui_64_mac): + addrs.append(addr) + + if addrs: + return addrs + + if fatal: + raise Exception("Interface '%s' does not have a scope global " + "non-temporary ipv6 address." % iface) + + return [] + + +def get_bridges(vnic_dir='/sys/devices/virtual/net'): + """Return a list of bridges on the system.""" + b_regex = "%s/*/bridge" % vnic_dir + return [x.replace(vnic_dir, '').split('/')[1] for x in glob.glob(b_regex)] + + +def get_bridge_nics(bridge, vnic_dir='/sys/devices/virtual/net'): + """Return a list of nics comprising a given bridge on the system.""" + brif_regex = "%s/%s/brif/*" % (vnic_dir, bridge) + return [x.split('/')[-1] for x in glob.glob(brif_regex)] + + +def is_bridge_member(nic): + """Check if a given nic is a member of a bridge.""" + for bridge in get_bridges(): + if nic in get_bridge_nics(bridge): + return True + + return False + + +def is_ip(address): + """ + Returns True if address is a valid IP address. + """ + try: + # Test to see if already an IPv4/IPv6 address + address = netaddr.IPAddress(address) + return True + except (netaddr.AddrFormatError, ValueError): + return False + + +def ns_query(address): + try: + import dns.resolver + except ImportError: + if six.PY2: + apt_install('python-dnspython', fatal=True) + else: + apt_install('python3-dnspython', fatal=True) + import dns.resolver + + if isinstance(address, dns.name.Name): + rtype = 'PTR' + elif isinstance(address, six.string_types): + rtype = 'A' + else: + return None + + try: + answers = dns.resolver.query(address, rtype) + except dns.resolver.NXDOMAIN: + return None + + if answers: + return str(answers[0]) + return None + + +def get_host_ip(hostname, fallback=None): + """ + Resolves the IP for a given hostname, or returns + the input if it is already an IP. + """ + if is_ip(hostname): + return hostname + + ip_addr = ns_query(hostname) + if not ip_addr: + try: + ip_addr = socket.gethostbyname(hostname) + except: + log("Failed to resolve hostname '%s'" % (hostname), + level=WARNING) + return fallback + return ip_addr + + +def get_hostname(address, fqdn=True): + """ + Resolves hostname for given IP, or returns the input + if it is already a hostname. + """ + if is_ip(address): + try: + import dns.reversename + except ImportError: + if six.PY2: + apt_install("python-dnspython", fatal=True) + else: + apt_install("python3-dnspython", fatal=True) + import dns.reversename + + rev = dns.reversename.from_address(address) + result = ns_query(rev) + + if not result: + try: + result = socket.gethostbyaddr(address)[0] + except: + return None + else: + result = address + + if fqdn: + # strip trailing . + if result.endswith('.'): + return result[:-1] + else: + return result + else: + return result.split('.')[0] + + +def port_has_listener(address, port): + """ + Returns True if the address:port is open and being listened to, + else False. + + @param address: an IP address or hostname + @param port: integer port + + Note calls 'zc' via a subprocess shell + """ + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not(bool(result)) + + +def assert_charm_supports_ipv6(): + """Check whether we are able to support charms ipv6.""" + release = lsb_release()['DISTRIB_CODENAME'].lower() + if CompareHostReleases(release) < "trusty": + raise Exception("IPv6 is not supported in the charms for Ubuntu " + "versions less than Trusty 14.04") + + +def get_relation_ip(interface, cidr_network=None): + """Return this unit's IP for the given interface. + + Allow for an arbitrary interface to use with network-get to select an IP. + Handle all address selection options including passed cidr network and + IPv6. + + Usage: get_relation_ip('amqp', cidr_network='10.0.0.0/8') + + @param interface: string name of the relation. + @param cidr_network: string CIDR Network to select an address from. + @raises Exception if prefer-ipv6 is configured but IPv6 unsupported. + @returns IPv6 or IPv4 address + """ + # Select the interface address first + # For possible use as a fallback bellow with get_address_in_network + try: + # Get the interface specific IP + address = network_get_primary_address(interface) + except NotImplementedError: + # If network-get is not available + address = get_host_ip(unit_get('private-address')) + + if config('prefer-ipv6'): + # Currently IPv6 has priority, eventually we want IPv6 to just be + # another network space. + assert_charm_supports_ipv6() + return get_ipv6_addr()[0] + elif cidr_network: + # If a specific CIDR network is passed get the address from that + # network. + return get_address_in_network(cidr_network, address) + + # Return the interface address + return address diff --git a/contrail-analytics/hooks/charmhelpers/contrib/network/ovs/__init__.py b/contrail-analytics/hooks/charmhelpers/contrib/network/ovs/__init__.py new file mode 100644 index 0000000..f044b60 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -0,0 +1,163 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' Helpers for interacting with OpenvSwitch ''' +import subprocess +import os +import six + +from charmhelpers.fetch import apt_install + + +from charmhelpers.core.hookenv import ( + log, WARNING, INFO, DEBUG +) +from charmhelpers.core.host import ( + service +) + +BRIDGE_TEMPLATE = """\ +# This veth pair is required when neutron data-port is mapped to an existing linux bridge. lp:1635067 + +auto {linuxbridge_port} +iface {linuxbridge_port} inet manual + pre-up ip link add name {linuxbridge_port} type veth peer name {ovsbridge_port} + pre-up ip link set {ovsbridge_port} master {bridge} + pre-up ip link set {ovsbridge_port} up + up ip link set {linuxbridge_port} up + down ip link del {linuxbridge_port} +""" + + +def add_bridge(name, datapath_type=None): + ''' Add the named bridge to openvswitch ''' + log('Creating bridge {}'.format(name)) + cmd = ["ovs-vsctl", "--", "--may-exist", "add-br", name] + if datapath_type is not None: + cmd += ['--', 'set', 'bridge', name, + 'datapath_type={}'.format(datapath_type)] + subprocess.check_call(cmd) + + +def del_bridge(name): + ''' Delete the named bridge from openvswitch ''' + log('Deleting bridge {}'.format(name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-br", name]) + + +def add_bridge_port(name, port, promisc=False): + ''' Add a port to the named openvswitch bridge ''' + log('Adding port {} to bridge {}'.format(port, name)) + subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-port", + name, port]) + subprocess.check_call(["ip", "link", "set", port, "up"]) + if promisc: + subprocess.check_call(["ip", "link", "set", port, "promisc", "on"]) + else: + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) + + +def del_bridge_port(name, port): + ''' Delete a port from the named openvswitch bridge ''' + log('Deleting port {} from bridge {}'.format(port, name)) + subprocess.check_call(["ovs-vsctl", "--", "--if-exists", "del-port", + name, port]) + subprocess.check_call(["ip", "link", "set", port, "down"]) + subprocess.check_call(["ip", "link", "set", port, "promisc", "off"]) + + +def add_ovsbridge_linuxbridge(name, bridge): + ''' Add linux bridge to the named openvswitch bridge + :param name: Name of ovs bridge to be added to Linux bridge + :param bridge: Name of Linux bridge to be added to ovs bridge + :returns: True if veth is added between ovs bridge and linux bridge, + False otherwise''' + try: + import netifaces + except ImportError: + if six.PY2: + apt_install('python-netifaces', fatal=True) + else: + apt_install('python3-netifaces', fatal=True) + import netifaces + + ovsbridge_port = "veth-" + name + linuxbridge_port = "veth-" + bridge + log('Adding linuxbridge {} to ovsbridge {}'.format(bridge, name), + level=INFO) + interfaces = netifaces.interfaces() + for interface in interfaces: + if interface == ovsbridge_port or interface == linuxbridge_port: + log('Interface {} already exists'.format(interface), level=INFO) + return + + with open('/etc/network/interfaces.d/{}.cfg'.format( + linuxbridge_port), 'w') as config: + config.write(BRIDGE_TEMPLATE.format(linuxbridge_port=linuxbridge_port, + ovsbridge_port=ovsbridge_port, + bridge=bridge)) + + subprocess.check_call(["ifup", linuxbridge_port]) + add_bridge_port(name, linuxbridge_port) + + +def is_linuxbridge_interface(port): + ''' Check if the interface is a linuxbridge bridge + :param port: Name of an interface to check whether it is a Linux bridge + :returns: True if port is a Linux bridge''' + + if os.path.exists('/sys/class/net/' + port + '/bridge'): + log('Interface {} is a Linux bridge'.format(port), level=DEBUG) + return True + else: + log('Interface {} is not a Linux bridge'.format(port), level=DEBUG) + return False + + +def set_manager(manager): + ''' Set the controller for the local openvswitch ''' + log('Setting manager for local ovs to {}'.format(manager)) + subprocess.check_call(['ovs-vsctl', 'set-manager', + 'ssl:{}'.format(manager)]) + + +CERT_PATH = '/etc/openvswitch/ovsclient-cert.pem' + + +def get_certificate(): + ''' Read openvswitch certificate from disk ''' + if os.path.exists(CERT_PATH): + log('Reading ovs certificate from {}'.format(CERT_PATH)) + with open(CERT_PATH, 'r') as cert: + full_cert = cert.read() + begin_marker = "-----BEGIN CERTIFICATE-----" + end_marker = "-----END CERTIFICATE-----" + begin_index = full_cert.find(begin_marker) + end_index = full_cert.rfind(end_marker) + if end_index == -1 or begin_index == -1: + raise RuntimeError("Certificate does not contain valid begin" + " and end markers.") + full_cert = full_cert[begin_index:(end_index + len(end_marker))] + return full_cert + else: + log('Certificate not found', level=WARNING) + return None + + +def full_restart(): + ''' Full restart and reload of openvswitch ''' + if os.path.exists('/etc/init/openvswitch-force-reload-kmod.conf'): + service('start', 'openvswitch-force-reload-kmod') + else: + service('force-reload-kmod', 'openvswitch-switch') diff --git a/contrail-analytics/hooks/charmhelpers/contrib/network/ufw.py b/contrail-analytics/hooks/charmhelpers/contrib/network/ufw.py new file mode 100644 index 0000000..5cff71b --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/contrib/network/ufw.py @@ -0,0 +1,316 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module contains helpers to add and remove ufw rules. + +Examples: + +- open SSH port for subnet 10.0.3.0/24: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.grant_access(src='10.0.3.0/24', dst='any', port='22', proto='tcp') + +- open service by name as defined in /etc/services: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('ssh', 'open') + +- close service by port number: + + >>> from charmhelpers.contrib.network import ufw + >>> ufw.enable() + >>> ufw.service('4949', 'close') # munin +""" +import re +import os +import subprocess + +from charmhelpers.core import hookenv +from charmhelpers.core.kernel import modprobe, is_module_loaded + +__author__ = "Felipe Reyes <felipe.reyes@canonical.com>" + + +class UFWError(Exception): + pass + + +class UFWIPv6Error(UFWError): + pass + + +def is_enabled(): + """ + Check if `ufw` is enabled + + :returns: True if ufw is enabled + """ + output = subprocess.check_output(['ufw', 'status'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall(r'^Status: active\n', output, re.M) + + return len(m) >= 1 + + +def is_ipv6_ok(soft_fail=False): + """ + Check if IPv6 support is present and ip6tables functional + + :param soft_fail: If set to True and IPv6 support is broken, then reports + that the host doesn't have IPv6 support, otherwise a + UFWIPv6Error exception is raised. + :returns: True if IPv6 is working, False otherwise + """ + + # do we have IPv6 in the machine? + if os.path.isdir('/proc/sys/net/ipv6'): + # is ip6tables kernel module loaded? + if not is_module_loaded('ip6_tables'): + # ip6tables support isn't complete, let's try to load it + try: + modprobe('ip6_tables') + # great, we can load the module + return True + except subprocess.CalledProcessError as ex: + hookenv.log("Couldn't load ip6_tables module: %s" % ex.output, + level="WARN") + # we are in a world where ip6tables isn't working + if soft_fail: + # so we inform that the machine doesn't have IPv6 + return False + else: + raise UFWIPv6Error("IPv6 firewall support broken") + else: + # the module is present :) + return True + + else: + # the system doesn't have IPv6 + return False + + +def disable_ipv6(): + """ + Disable ufw IPv6 support in /etc/default/ufw + """ + exit_code = subprocess.call(['sed', '-i', 's/IPV6=.*/IPV6=no/g', + '/etc/default/ufw']) + if exit_code == 0: + hookenv.log('IPv6 support in ufw disabled', level='INFO') + else: + hookenv.log("Couldn't disable IPv6 support in ufw", level="ERROR") + raise UFWError("Couldn't disable IPv6 support in ufw") + + +def enable(soft_fail=False): + """ + Enable ufw + + :param soft_fail: If set to True silently disables IPv6 support in ufw, + otherwise a UFWIPv6Error exception is raised when IP6 + support is broken. + :returns: True if ufw is successfully enabled + """ + if is_enabled(): + return True + + if not is_ipv6_ok(soft_fail): + disable_ipv6() + + output = subprocess.check_output(['ufw', 'enable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall('^Firewall is active and enabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be enabled", level='WARN') + return False + else: + hookenv.log("ufw enabled", level='INFO') + return True + + +def disable(): + """ + Disable ufw + + :returns: True if ufw is successfully disabled + """ + if not is_enabled(): + return True + + output = subprocess.check_output(['ufw', 'disable'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall(r'^Firewall stopped and disabled on system startup\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be disabled", level='WARN') + return False + else: + hookenv.log("ufw disabled", level='INFO') + return True + + +def default_policy(policy='deny', direction='incoming'): + """ + Changes the default policy for traffic `direction` + + :param policy: allow, deny or reject + :param direction: traffic direction, possible values: incoming, outgoing, + routed + """ + if policy not in ['allow', 'deny', 'reject']: + raise UFWError(('Unknown policy %s, valid values: ' + 'allow, deny, reject') % policy) + + if direction not in ['incoming', 'outgoing', 'routed']: + raise UFWError(('Unknown direction %s, valid values: ' + 'incoming, outgoing, routed') % direction) + + output = subprocess.check_output(['ufw', 'default', policy, direction], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + hookenv.log(output, level='DEBUG') + + m = re.findall("^Default %s policy changed to '%s'\n" % (direction, + policy), + output, re.M) + if len(m) == 0: + hookenv.log("ufw couldn't change the default policy to %s for %s" + % (policy, direction), level='WARN') + return False + else: + hookenv.log("ufw default policy for %s changed to %s" + % (direction, policy), level='INFO') + return True + + +def modify_access(src, dst='any', port=None, proto=None, action='allow', + index=None): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + :param action: `allow` or `delete` + :param index: if different from None the rule is inserted at the given + `index`. + """ + if not is_enabled(): + hookenv.log('ufw is disabled, skipping modify_access()', level='WARN') + return + + if action == 'delete': + cmd = ['ufw', 'delete', 'allow'] + elif index is not None: + cmd = ['ufw', 'insert', str(index), action] + else: + cmd = ['ufw', action] + + if src is not None: + cmd += ['from', src] + + if dst is not None: + cmd += ['to', dst] + + if port is not None: + cmd += ['port', str(port)] + + if proto is not None: + cmd += ['proto', proto] + + hookenv.log('ufw {}: {}'.format(action, ' '.join(cmd)), level='DEBUG') + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + (stdout, stderr) = p.communicate() + + hookenv.log(stdout, level='INFO') + + if p.returncode != 0: + hookenv.log(stderr, level='ERROR') + hookenv.log('Error running: {}, exit code: {}'.format(' '.join(cmd), + p.returncode), + level='ERROR') + + +def grant_access(src, dst='any', port=None, proto=None, index=None): + """ + Grant access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + :param index: if different from None the rule is inserted at the given + `index`. + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='allow', + index=index) + + +def revoke_access(src, dst='any', port=None, proto=None): + """ + Revoke access to an address or subnet + + :param src: address (e.g. 192.168.1.234) or subnet + (e.g. 192.168.1.0/24). + :param dst: destiny of the connection, if the machine has multiple IPs and + connections to only one of those have to accepted this is the + field has to be set. + :param port: destiny port + :param proto: protocol (tcp or udp) + """ + return modify_access(src, dst=dst, port=port, proto=proto, action='delete') + + +def service(name, action): + """ + Open/close access to a service + + :param name: could be a service name defined in `/etc/services` or a port + number. + :param action: `open` or `close` + """ + if action == 'open': + subprocess.check_output(['ufw', 'allow', str(name)], + universal_newlines=True) + elif action == 'close': + subprocess.check_output(['ufw', 'delete', 'allow', str(name)], + universal_newlines=True) + else: + raise UFWError(("'{}' not supported, use 'allow' " + "or 'delete'").format(action)) diff --git a/contrail-analytics/hooks/charmhelpers/core/__init__.py b/contrail-analytics/hooks/charmhelpers/core/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/contrail-analytics/hooks/charmhelpers/core/decorators.py b/contrail-analytics/hooks/charmhelpers/core/decorators.py new file mode 100644 index 0000000..6ad41ee --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/decorators.py @@ -0,0 +1,55 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Copyright 2014 Canonical Ltd. +# +# Authors: +# Edward Hope-Morley <opentastic@gmail.com> +# + +import time + +from charmhelpers.core.hookenv import ( + log, + INFO, +) + + +def retry_on_exception(num_retries, base_delay=0, exc_type=Exception): + """If the decorated function raises exception exc_type, allow num_retries + retry attempts before raise the exception. + """ + def _retry_on_exception_inner_1(f): + def _retry_on_exception_inner_2(*args, **kwargs): + retries = num_retries + multiplier = 1 + while True: + try: + return f(*args, **kwargs) + except exc_type: + if not retries: + raise + + delay = base_delay * multiplier + multiplier += 1 + log("Retrying '%s' %d more times (delay=%s)" % + (f.__name__, retries, delay), level=INFO) + retries -= 1 + if delay: + time.sleep(delay) + + return _retry_on_exception_inner_2 + + return _retry_on_exception_inner_1 diff --git a/contrail-analytics/hooks/charmhelpers/core/files.py b/contrail-analytics/hooks/charmhelpers/core/files.py new file mode 100644 index 0000000..fdd82b7 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/files.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>' + +import os +import subprocess + + +def sed(filename, before, after, flags='g'): + """ + Search and replaces the given pattern on filename. + + :param filename: relative or absolute file path. + :param before: expression to be replaced (see 'man sed') + :param after: expression to replace with (see 'man sed') + :param flags: sed-compatible regex flags in example, to make + the search and replace case insensitive, specify ``flags="i"``. + The ``g`` flag is always specified regardless, so you do not + need to remember to include it when overriding this parameter. + :returns: If the sed command exit code was zero then return, + otherwise raise CalledProcessError. + """ + expression = r's/{0}/{1}/{2}'.format(before, + after, flags) + + return subprocess.check_call(["sed", "-i", "-r", "-e", + expression, + os.path.expanduser(filename)]) diff --git a/contrail-analytics/hooks/charmhelpers/core/fstab.py b/contrail-analytics/hooks/charmhelpers/core/fstab.py new file mode 100644 index 0000000..d9fa915 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/fstab.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os + +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' + + +class Fstab(io.FileIO): + """This class extends file in order to implement a file reader/writer + for file `/etc/fstab` + """ + + class Entry(object): + """Entry class represents a non-comment line on the `/etc/fstab` file + """ + def __init__(self, device, mountpoint, filesystem, + options, d=0, p=0): + self.device = device + self.mountpoint = mountpoint + self.filesystem = filesystem + + if not options: + options = "defaults" + + self.options = options + self.d = int(d) + self.p = int(p) + + def __eq__(self, o): + return str(self) == str(o) + + def __str__(self): + return "{} {} {} {} {} {}".format(self.device, + self.mountpoint, + self.filesystem, + self.options, + self.d, + self.p) + + DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab') + + def __init__(self, path=None): + if path: + self._path = path + else: + self._path = self.DEFAULT_PATH + super(Fstab, self).__init__(self._path, 'rb+') + + def _hydrate_entry(self, line): + # NOTE: use split with no arguments to split on any + # whitespace including tabs + return Fstab.Entry(*filter( + lambda x: x not in ('', None), + line.strip("\n").split())) + + @property + def entries(self): + self.seek(0) + for line in self.readlines(): + line = line.decode('us-ascii') + try: + if line.strip() and not line.strip().startswith("#"): + yield self._hydrate_entry(line) + except ValueError: + pass + + def get_entry_by_attr(self, attr, value): + for entry in self.entries: + e_attr = getattr(entry, attr) + if e_attr == value: + return entry + return None + + def add_entry(self, entry): + if self.get_entry_by_attr('device', entry.device): + return False + + self.write((str(entry) + '\n').encode('us-ascii')) + self.truncate() + return entry + + def remove_entry(self, entry): + self.seek(0) + + lines = [l.decode('us-ascii') for l in self.readlines()] + + found = False + for index, line in enumerate(lines): + if line.strip() and not line.strip().startswith("#"): + if self._hydrate_entry(line) == entry: + found = True + break + + if not found: + return False + + lines.remove(line) + + self.seek(0) + self.write(''.join(lines).encode('us-ascii')) + self.truncate() + return True + + @classmethod + def remove_by_mountpoint(cls, mountpoint, path=None): + fstab = cls(path=path) + entry = fstab.get_entry_by_attr('mountpoint', mountpoint) + if entry: + return fstab.remove_entry(entry) + return False + + @classmethod + def add(cls, device, mountpoint, filesystem, options=None, path=None): + return cls(path=path).add_entry(Fstab.Entry(device, + mountpoint, filesystem, + options=options)) diff --git a/contrail-analytics/hooks/charmhelpers/core/hookenv.py b/contrail-analytics/hooks/charmhelpers/core/hookenv.py new file mode 100644 index 0000000..e44e22b --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/hookenv.py @@ -0,0 +1,1068 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"Interactions with the Juju environment" +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers <juju@lists.ubuntu.com> + +from __future__ import print_function +import copy +from distutils.version import LooseVersion +from functools import wraps +import glob +import os +import json +import yaml +import subprocess +import sys +import errno +import tempfile +from subprocess import CalledProcessError + +import six +if not six.PY3: + from UserDict import UserDict +else: + from collections import UserDict + +CRITICAL = "CRITICAL" +ERROR = "ERROR" +WARNING = "WARNING" +INFO = "INFO" +DEBUG = "DEBUG" +MARKER = object() + +cache = {} + + +def cached(func): + """Cache return values for multiple executions of func + args + + For example:: + + @cached + def unit_get(attribute): + pass + + unit_get('test') + + will cache the result of unit_get + 'test' for future calls. + """ + @wraps(func) + def wrapper(*args, **kwargs): + global cache + key = str((func, args, kwargs)) + try: + return cache[key] + except KeyError: + pass # Drop out of the exception handler scope. + res = func(*args, **kwargs) + cache[key] = res + return res + wrapper._wrapped = func + return wrapper + + +def flush(key): + """Flushes any entries from function cache where the + key is found in the function+args """ + flush_list = [] + for item in cache: + if key in item: + flush_list.append(item) + for item in flush_list: + del cache[item] + + +def log(message, level=None): + """Write a message to the juju log""" + command = ['juju-log'] + if level: + command += ['-l', level] + if not isinstance(message, six.string_types): + message = repr(message) + command += [message] + # Missing juju-log should not cause failures in unit tests + # Send log output to stderr + try: + subprocess.call(command) + except OSError as e: + if e.errno == errno.ENOENT: + if level: + message = "{}: {}".format(level, message) + message = "juju-log: {}".format(message) + print(message, file=sys.stderr) + else: + raise + + +class Serializable(UserDict): + """Wrapper, an object that can be serialized to yaml or json""" + + def __init__(self, obj): + # wrap the object + UserDict.__init__(self) + self.data = obj + + def __getattr__(self, attr): + # See if this object has attribute. + if attr in ("json", "yaml", "data"): + return self.__dict__[attr] + # Check for attribute in wrapped object. + got = getattr(self.data, attr, MARKER) + if got is not MARKER: + return got + # Proxy to the wrapped object via dict interface. + try: + return self.data[attr] + except KeyError: + raise AttributeError(attr) + + def __getstate__(self): + # Pickle as a standard dictionary. + return self.data + + def __setstate__(self, state): + # Unpickle into our wrapper. + self.data = state + + def json(self): + """Serialize the object to json""" + return json.dumps(self.data) + + def yaml(self): + """Serialize the object to yaml""" + return yaml.dump(self.data) + + +def execution_environment(): + """A convenient bundling of the current execution context""" + context = {} + context['conf'] = config() + if relation_id(): + context['reltype'] = relation_type() + context['relid'] = relation_id() + context['rel'] = relation_get() + context['unit'] = local_unit() + context['rels'] = relations() + context['env'] = os.environ + return context + + +def in_relation_hook(): + """Determine whether we're running in a relation hook""" + return 'JUJU_RELATION' in os.environ + + +def relation_type(): + """The scope for the current relation hook""" + return os.environ.get('JUJU_RELATION', None) + + +@cached +def relation_id(relation_name=None, service_or_unit=None): + """The relation ID for the current or a specified relation""" + if not relation_name and not service_or_unit: + return os.environ.get('JUJU_RELATION_ID', None) + elif relation_name and service_or_unit: + service_name = service_or_unit.split('/')[0] + for relid in relation_ids(relation_name): + remote_service = remote_service_name(relid) + if remote_service == service_name: + return relid + else: + raise ValueError('Must specify neither or both of relation_name and service_or_unit') + + +def local_unit(): + """Local unit ID""" + return os.environ['JUJU_UNIT_NAME'] + + +def remote_unit(): + """The remote unit for the current relation hook""" + return os.environ.get('JUJU_REMOTE_UNIT', None) + + +def service_name(): + """The name service group this unit belongs to""" + return local_unit().split('/')[0] + + +@cached +def remote_service_name(relid=None): + """The remote service name for a given relation-id (or the current relation)""" + if relid is None: + unit = remote_unit() + else: + units = related_units(relid) + unit = units[0] if units else None + return unit.split('/')[0] if unit else None + + +def hook_name(): + """The name of the currently executing hook""" + return os.environ.get('JUJU_HOOK_NAME', os.path.basename(sys.argv[0])) + + +class Config(dict): + """A dictionary representation of the charm's config.yaml, with some + extra features: + + - See which values in the dictionary have changed since the previous hook. + - For values that have changed, see what the previous value was. + - Store arbitrary data for use in a later hook. + + NOTE: Do not instantiate this object directly - instead call + ``hookenv.config()``, which will return an instance of :class:`Config`. + + Example usage:: + + >>> # inside a hook + >>> from charmhelpers.core import hookenv + >>> config = hookenv.config() + >>> config['foo'] + 'bar' + >>> # store a new key/value for later use + >>> config['mykey'] = 'myval' + + + >>> # user runs `juju set mycharm foo=baz` + >>> # now we're inside subsequent config-changed hook + >>> config = hookenv.config() + >>> config['foo'] + 'baz' + >>> # test to see if this val has changed since last hook + >>> config.changed('foo') + True + >>> # what was the previous value? + >>> config.previous('foo') + 'bar' + >>> # keys/values that we add are preserved across hooks + >>> config['mykey'] + 'myval' + + """ + CONFIG_FILE_NAME = '.juju-persistent-config' + + def __init__(self, *args, **kw): + super(Config, self).__init__(*args, **kw) + self.implicit_save = True + self._prev_dict = None + self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) + if os.path.exists(self.path): + self.load_previous() + atexit(self._implicit_save) + + def load_previous(self, path=None): + """Load previous copy of config from disk. + + In normal usage you don't need to call this method directly - it + is called automatically at object initialization. + + :param path: + + File path from which to load the previous config. If `None`, + config is loaded from the default location. If `path` is + specified, subsequent `save()` calls will write to the same + path. + + """ + self.path = path or self.path + with open(self.path) as f: + self._prev_dict = json.load(f) + for k, v in copy.deepcopy(self._prev_dict).items(): + if k not in self: + self[k] = v + + def changed(self, key): + """Return True if the current value for this key is different from + the previous value. + + """ + if self._prev_dict is None: + return True + return self.previous(key) != self.get(key) + + def previous(self, key): + """Return previous value for this key, or None if there + is no previous value. + + """ + if self._prev_dict: + return self._prev_dict.get(key) + return None + + def save(self): + """Save this config to disk. + + If the charm is using the :mod:`Services Framework <services.base>` + or :meth:'@hook <Hooks.hook>' decorator, this + is called automatically at the end of successful hook execution. + Otherwise, it should be called directly by user code. + + To disable automatic saves, set ``implicit_save=False`` on this + instance. + + """ + with open(self.path, 'w') as f: + json.dump(self, f) + + def _implicit_save(self): + if self.implicit_save: + self.save() + + +@cached +def config(scope=None): + """Juju charm configuration""" + config_cmd_line = ['config-get'] + if scope is not None: + config_cmd_line.append(scope) + else: + config_cmd_line.append('--all') + config_cmd_line.append('--format=json') + try: + config_data = json.loads( + subprocess.check_output(config_cmd_line).decode('UTF-8')) + if scope is not None: + return config_data + return Config(config_data) + except ValueError: + return None + + +@cached +def relation_get(attribute=None, unit=None, rid=None): + """Get relation information""" + _args = ['relation-get', '--format=json'] + if rid: + _args.append('-r') + _args.append(rid) + _args.append(attribute or '-') + if unit: + _args.append(unit) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + except CalledProcessError as e: + if e.returncode == 2: + return None + raise + + +def relation_set(relation_id=None, relation_settings=None, **kwargs): + """Set relation information for the current unit""" + relation_settings = relation_settings if relation_settings else {} + relation_cmd_line = ['relation-set'] + accepts_file = "--file" in subprocess.check_output( + relation_cmd_line + ["--help"], universal_newlines=True) + if relation_id is not None: + relation_cmd_line.extend(('-r', relation_id)) + settings = relation_settings.copy() + settings.update(kwargs) + for key, value in settings.items(): + # Force value to be a string: it always should, but some call + # sites pass in things like dicts or numbers. + if value is not None: + settings[key] = "{}".format(value) + if accepts_file: + # --file was introduced in Juju 1.23.2. Use it by default if + # available, since otherwise we'll break if the relation data is + # too big. Ideally we should tell relation-set to read the data from + # stdin, but that feature is broken in 1.23.2: Bug #1454678. + with tempfile.NamedTemporaryFile(delete=False) as settings_file: + settings_file.write(yaml.safe_dump(settings).encode("utf-8")) + subprocess.check_call( + relation_cmd_line + ["--file", settings_file.name]) + os.remove(settings_file.name) + else: + for key, value in settings.items(): + if value is None: + relation_cmd_line.append('{}='.format(key)) + else: + relation_cmd_line.append('{}={}'.format(key, value)) + subprocess.check_call(relation_cmd_line) + # Flush cache of any relation-gets for local unit + flush(local_unit()) + + +def relation_clear(r_id=None): + ''' Clears any relation data already set on relation r_id ''' + settings = relation_get(rid=r_id, + unit=local_unit()) + for setting in settings: + if setting not in ['public-address', 'private-address']: + settings[setting] = None + relation_set(relation_id=r_id, + **settings) + + +@cached +def relation_ids(reltype=None): + """A list of relation_ids""" + reltype = reltype or relation_type() + relid_cmd_line = ['relation-ids', '--format=json'] + if reltype is not None: + relid_cmd_line.append(reltype) + return json.loads( + subprocess.check_output(relid_cmd_line).decode('UTF-8')) or [] + return [] + + +@cached +def related_units(relid=None): + """A list of related units""" + relid = relid or relation_id() + units_cmd_line = ['relation-list', '--format=json'] + if relid is not None: + units_cmd_line.extend(('-r', relid)) + return json.loads( + subprocess.check_output(units_cmd_line).decode('UTF-8')) or [] + + +@cached +def relation_for_unit(unit=None, rid=None): + """Get the json represenation of a unit's relation""" + unit = unit or remote_unit() + relation = relation_get(unit=unit, rid=rid) + for key in relation: + if key.endswith('-list'): + relation[key] = relation[key].split() + relation['__unit__'] = unit + return relation + + +@cached +def relations_for_id(relid=None): + """Get relations of a specific relation ID""" + relation_data = [] + relid = relid or relation_ids() + for unit in related_units(relid): + unit_data = relation_for_unit(unit, relid) + unit_data['__relid__'] = relid + relation_data.append(unit_data) + return relation_data + + +@cached +def relations_of_type(reltype=None): + """Get relations of a specific type""" + relation_data = [] + reltype = reltype or relation_type() + for relid in relation_ids(reltype): + for relation in relations_for_id(relid): + relation['__relid__'] = relid + relation_data.append(relation) + return relation_data + + +@cached +def metadata(): + """Get the current charm metadata.yaml contents as a python object""" + with open(os.path.join(charm_dir(), 'metadata.yaml')) as md: + return yaml.safe_load(md) + + +@cached +def relation_types(): + """Get a list of relation types supported by this charm""" + rel_types = [] + md = metadata() + for key in ('provides', 'requires', 'peers'): + section = md.get(key) + if section: + rel_types.extend(section.keys()) + return rel_types + + +@cached +def peer_relation_id(): + '''Get the peers relation id if a peers relation has been joined, else None.''' + md = metadata() + section = md.get('peers') + if section: + for key in section: + relids = relation_ids(key) + if relids: + return relids[0] + return None + + +@cached +def relation_to_interface(relation_name): + """ + Given the name of a relation, return the interface that relation uses. + + :returns: The interface name, or ``None``. + """ + return relation_to_role_and_interface(relation_name)[1] + + +@cached +def relation_to_role_and_interface(relation_name): + """ + Given the name of a relation, return the role and the name of the interface + that relation uses (where role is one of ``provides``, ``requires``, or ``peers``). + + :returns: A tuple containing ``(role, interface)``, or ``(None, None)``. + """ + _metadata = metadata() + for role in ('provides', 'requires', 'peers'): + interface = _metadata.get(role, {}).get(relation_name, {}).get('interface') + if interface: + return role, interface + return None, None + + +@cached +def role_and_interface_to_relations(role, interface_name): + """ + Given a role and interface name, return a list of relation names for the + current charm that use that interface under that role (where role is one + of ``provides``, ``requires``, or ``peers``). + + :returns: A list of relation names. + """ + _metadata = metadata() + results = [] + for relation_name, relation in _metadata.get(role, {}).items(): + if relation['interface'] == interface_name: + results.append(relation_name) + return results + + +@cached +def interface_to_relations(interface_name): + """ + Given an interface, return a list of relation names for the current + charm that use that interface. + + :returns: A list of relation names. + """ + results = [] + for role in ('provides', 'requires', 'peers'): + results.extend(role_and_interface_to_relations(role, interface_name)) + return results + + +@cached +def charm_name(): + """Get the name of the current charm as is specified on metadata.yaml""" + return metadata().get('name') + + +@cached +def relations(): + """Get a nested dictionary of relation data for all related units""" + rels = {} + for reltype in relation_types(): + relids = {} + for relid in relation_ids(reltype): + units = {local_unit(): relation_get(unit=local_unit(), rid=relid)} + for unit in related_units(relid): + reldata = relation_get(unit=unit, rid=relid) + units[unit] = reldata + relids[relid] = units + rels[reltype] = relids + return rels + + +@cached +def is_relation_made(relation, keys='private-address'): + ''' + Determine whether a relation is established by checking for + presence of key(s). If a list of keys is provided, they + must all be present for the relation to be identified as made + ''' + if isinstance(keys, str): + keys = [keys] + for r_id in relation_ids(relation): + for unit in related_units(r_id): + context = {} + for k in keys: + context[k] = relation_get(k, rid=r_id, + unit=unit) + if None not in context.values(): + return True + return False + + +def open_port(port, protocol="TCP"): + """Open a service network port""" + _args = ['open-port'] + _args.append('{}/{}'.format(port, protocol)) + subprocess.check_call(_args) + + +def close_port(port, protocol="TCP"): + """Close a service network port""" + _args = ['close-port'] + _args.append('{}/{}'.format(port, protocol)) + subprocess.check_call(_args) + + +def open_ports(start, end, protocol="TCP"): + """Opens a range of service network ports""" + _args = ['open-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +def close_ports(start, end, protocol="TCP"): + """Close a range of service network ports""" + _args = ['close-port'] + _args.append('{}-{}/{}'.format(start, end, protocol)) + subprocess.check_call(_args) + + +@cached +def unit_get(attribute): + """Get the unit ID for the remote unit""" + _args = ['unit-get', '--format=json', attribute] + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + + +def unit_public_ip(): + """Get this unit's public IP address""" + return unit_get('public-address') + + +def unit_private_ip(): + """Get this unit's private IP address""" + return unit_get('private-address') + + +@cached +def storage_get(attribute=None, storage_id=None): + """Get storage attributes""" + _args = ['storage-get', '--format=json'] + if storage_id: + _args.extend(('-s', storage_id)) + if attribute: + _args.append(attribute) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + + +@cached +def storage_list(storage_name=None): + """List the storage IDs for the unit""" + _args = ['storage-list', '--format=json'] + if storage_name: + _args.append(storage_name) + try: + return json.loads(subprocess.check_output(_args).decode('UTF-8')) + except ValueError: + return None + except OSError as e: + import errno + if e.errno == errno.ENOENT: + # storage-list does not exist + return [] + raise + + +class UnregisteredHookError(Exception): + """Raised when an undefined hook is called""" + pass + + +class Hooks(object): + """A convenient handler for hook functions. + + Example:: + + hooks = Hooks() + + # register a hook, taking its name from the function name + @hooks.hook() + def install(): + pass # your code here + + # register a hook, providing a custom hook name + @hooks.hook("config-changed") + def config_changed(): + pass # your code here + + if __name__ == "__main__": + # execute a hook based on the name the program is called by + hooks.execute(sys.argv) + """ + + def __init__(self, config_save=None): + super(Hooks, self).__init__() + self._hooks = {} + + # For unknown reasons, we allow the Hooks constructor to override + # config().implicit_save. + if config_save is not None: + config().implicit_save = config_save + + def register(self, name, function): + """Register a hook""" + self._hooks[name] = function + + def execute(self, args): + """Execute a registered hook based on args[0]""" + _run_atstart() + hook_name = os.path.basename(args[0]) + if hook_name in self._hooks: + try: + self._hooks[hook_name]() + except SystemExit as x: + if x.code is None or x.code == 0: + _run_atexit() + raise + _run_atexit() + else: + raise UnregisteredHookError(hook_name) + + def hook(self, *hook_names): + """Decorator, registering them as hooks""" + def wrapper(decorated): + for hook_name in hook_names: + self.register(hook_name, decorated) + else: + self.register(decorated.__name__, decorated) + if '_' in decorated.__name__: + self.register( + decorated.__name__.replace('_', '-'), decorated) + return decorated + return wrapper + + +def charm_dir(): + """Return the root directory of the current charm""" + return os.environ.get('CHARM_DIR') + + +@cached +def action_get(key=None): + """Gets the value of an action parameter, or all key/value param pairs""" + cmd = ['action-get'] + if key is not None: + cmd.append(key) + cmd.append('--format=json') + action_data = json.loads(subprocess.check_output(cmd).decode('UTF-8')) + return action_data + + +def action_set(values): + """Sets the values to be returned after the action finishes""" + cmd = ['action-set'] + for k, v in list(values.items()): + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +def action_fail(message): + """Sets the action status to failed and sets the error message. + + The results set by action_set are preserved.""" + subprocess.check_call(['action-fail', message]) + + +def action_name(): + """Get the name of the currently executing action.""" + return os.environ.get('JUJU_ACTION_NAME') + + +def action_uuid(): + """Get the UUID of the currently executing action.""" + return os.environ.get('JUJU_ACTION_UUID') + + +def action_tag(): + """Get the tag for the currently executing action.""" + return os.environ.get('JUJU_ACTION_TAG') + + +def status_set(workload_state, message): + """Set the workload state with a message + + Use status-set to set the workload state with a message which is visible + to the user via juju status. If the status-set command is not found then + assume this is juju < 1.23 and juju-log the message unstead. + + workload_state -- valid juju workload state. + message -- status update message + """ + valid_states = ['maintenance', 'blocked', 'waiting', 'active'] + if workload_state not in valid_states: + raise ValueError( + '{!r} is not a valid workload state'.format(workload_state) + ) + cmd = ['status-set', workload_state, message] + try: + ret = subprocess.call(cmd) + if ret == 0: + return + except OSError as e: + if e.errno != errno.ENOENT: + raise + log_message = 'status-set failed: {} {}'.format(workload_state, + message) + log(log_message, level='INFO') + + +def status_get(): + """Retrieve the previously set juju workload state and message + + If the status-get command is not found then assume this is juju < 1.23 and + return 'unknown', "" + + """ + cmd = ['status-get', "--format=json", "--include-data"] + try: + raw_status = subprocess.check_output(cmd) + except OSError as e: + if e.errno == errno.ENOENT: + return ('unknown', "") + else: + raise + else: + status = json.loads(raw_status.decode("UTF-8")) + return (status["status"], status["message"]) + + +def translate_exc(from_exc, to_exc): + def inner_translate_exc1(f): + @wraps(f) + def inner_translate_exc2(*args, **kwargs): + try: + return f(*args, **kwargs) + except from_exc: + raise to_exc + + return inner_translate_exc2 + + return inner_translate_exc1 + + +def application_version_set(version): + """Charm authors may trigger this command from any hook to output what + version of the application is running. This could be a package version, + for instance postgres version 9.5. It could also be a build number or + version control revision identifier, for instance git sha 6fb7ba68. """ + + cmd = ['application-version-set'] + cmd.append(version) + try: + subprocess.check_call(cmd) + except OSError: + log("Application Version: {}".format(version)) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def is_leader(): + """Does the current unit hold the juju leadership + + Uses juju to determine whether the current unit is the leader of its peers + """ + cmd = ['is-leader', '--format=json'] + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def leader_get(attribute=None): + """Juju leader get value(s)""" + cmd = ['leader-get', '--format=json'] + [attribute or '-'] + return json.loads(subprocess.check_output(cmd).decode('UTF-8')) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def leader_set(settings=None, **kwargs): + """Juju leader set value(s)""" + # Don't log secrets. + # log("Juju leader-set '%s'" % (settings), level=DEBUG) + cmd = ['leader-set'] + settings = settings or {} + settings.update(kwargs) + for k, v in settings.items(): + if v is None: + cmd.append('{}='.format(k)) + else: + cmd.append('{}={}'.format(k, v)) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_register(ptype, klass, pid): + """ is used while a hook is running to let Juju know that a + payload has been started.""" + cmd = ['payload-register'] + for x in [ptype, klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_unregister(klass, pid): + """ is used while a hook is running to let Juju know + that a payload has been manually stopped. The <class> and <id> provided + must match a payload that has been previously registered with juju using + payload-register.""" + cmd = ['payload-unregister'] + for x in [klass, pid]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def payload_status_set(klass, pid, status): + """is used to update the current status of a registered payload. + The <class> and <id> provided must match a payload that has been previously + registered with juju using payload-register. The <status> must be one of the + follow: starting, started, stopping, stopped""" + cmd = ['payload-status-set'] + for x in [klass, pid, status]: + cmd.append(x) + subprocess.check_call(cmd) + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def resource_get(name): + """used to fetch the resource path of the given name. + + <name> must match a name of defined resource in metadata.yaml + + returns either a path or False if resource not available + """ + if not name: + return False + + cmd = ['resource-get', name] + try: + return subprocess.check_output(cmd).decode('UTF-8') + except subprocess.CalledProcessError: + return False + + +@cached +def juju_version(): + """Full version string (eg. '1.23.3.1-trusty-amd64')""" + # Per https://bugs.launchpad.net/juju-core/+bug/1455368/comments/1 + jujud = glob.glob('/var/lib/juju/tools/machine-*/jujud')[0] + return subprocess.check_output([jujud, 'version'], + universal_newlines=True).strip() + + +@cached +def has_juju_version(minimum_version): + """Return True if the Juju version is at least the provided version""" + return LooseVersion(juju_version()) >= LooseVersion(minimum_version) + + +_atexit = [] +_atstart = [] + + +def atstart(callback, *args, **kwargs): + '''Schedule a callback to run before the main hook. + + Callbacks are run in the order they were added. + + This is useful for modules and classes to perform initialization + and inject behavior. In particular: + + - Run common code before all of your hooks, such as logging + the hook name or interesting relation data. + - Defer object or module initialization that requires a hook + context until we know there actually is a hook context, + making testing easier. + - Rather than requiring charm authors to include boilerplate to + invoke your helper's behavior, have it run automatically if + your object is instantiated or module imported. + + This is not at all useful after your hook framework as been launched. + ''' + global _atstart + _atstart.append((callback, args, kwargs)) + + +def atexit(callback, *args, **kwargs): + '''Schedule a callback to run on successful hook completion. + + Callbacks are run in the reverse order that they were added.''' + _atexit.append((callback, args, kwargs)) + + +def _run_atstart(): + '''Hook frameworks must invoke this before running the main hook body.''' + global _atstart + for callback, args, kwargs in _atstart: + callback(*args, **kwargs) + del _atstart[:] + + +def _run_atexit(): + '''Hook frameworks must invoke this after the main hook body has + successfully completed. Do not invoke it if the hook fails.''' + global _atexit + for callback, args, kwargs in reversed(_atexit): + callback(*args, **kwargs) + del _atexit[:] + + +@translate_exc(from_exc=OSError, to_exc=NotImplementedError) +def network_get_primary_address(binding): + ''' + Retrieve the primary network address for a named binding + + :param binding: string. The name of a relation of extra-binding + :return: string. The primary IP address for the named binding + :raise: NotImplementedError if run on Juju < 2.0 + ''' + cmd = ['network-get', '--primary-address', binding] + return subprocess.check_output(cmd).decode('UTF-8').strip() + + +def add_metric(*args, **kwargs): + """Add metric values. Values may be expressed with keyword arguments. For + metric names containing dashes, these may be expressed as one or more + 'key=value' positional arguments. May only be called from the collect-metrics + hook.""" + _args = ['add-metric'] + _kvpairs = [] + _kvpairs.extend(args) + _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()]) + _args.extend(sorted(_kvpairs)) + try: + subprocess.check_call(_args) + return + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs)) + log(log_message, level='INFO') + + +def meter_status(): + """Get the meter status, if running in the meter-status-changed hook.""" + return os.environ.get('JUJU_METER_STATUS') + + +def meter_info(): + """Get the meter status information, if running in the meter-status-changed + hook.""" + return os.environ.get('JUJU_METER_INFO') diff --git a/contrail-analytics/hooks/charmhelpers/core/host.py b/contrail-analytics/hooks/charmhelpers/core/host.py new file mode 100644 index 0000000..b0043cb --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/host.py @@ -0,0 +1,924 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tools for working with the host system""" +# Copyright 2012 Canonical Ltd. +# +# Authors: +# Nick Moffitt <nick.moffitt@canonical.com> +# Matthew Wedgwood <matthew.wedgwood@canonical.com> + +import os +import re +import pwd +import glob +import grp +import random +import string +import subprocess +import hashlib +import functools +import itertools +import six + +from contextlib import contextmanager +from collections import OrderedDict +from .hookenv import log +from .fstab import Fstab +from charmhelpers.osplatform import get_platform + +__platform__ = get_platform() +if __platform__ == "ubuntu": + from charmhelpers.core.host_factory.ubuntu import ( + service_available, + add_new_group, + lsb_release, + cmp_pkgrevno, + CompareHostReleases, + ) # flake8: noqa -- ignore F401 for this import +elif __platform__ == "centos": + from charmhelpers.core.host_factory.centos import ( + service_available, + add_new_group, + lsb_release, + cmp_pkgrevno, + CompareHostReleases, + ) # flake8: noqa -- ignore F401 for this import + +UPDATEDB_PATH = '/etc/updatedb.conf' + +def service_start(service_name, **kwargs): + """Start a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('start', service_name, **kwargs) + + +def service_stop(service_name, **kwargs): + """Stop a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example stops the ceph-osd service for instance id=4: + + service_stop('ceph-osd', id=4) + + :param service_name: the name of the service to stop + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + return service('stop', service_name, **kwargs) + + +def service_restart(service_name, **kwargs): + """Restart a system service. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be restarted. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_restart('ceph-osd', id=4) + + :param service_name: the name of the service to restart + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + return service('restart', service_name) + + +def service_reload(service_name, restart_on_failure=False, **kwargs): + """Reload a system service, optionally falling back to restart if + reload fails. + + The specified service name is managed via the system level init system. + Some init systems (e.g. upstart) require that additional arguments be + provided in order to directly control service instances whereas other init + systems allow for addressing instances of a service directly by name (e.g. + systemd). + + The kwargs allow for the additional parameters to be passed to underlying + init systems for those systems which require/allow for them. For example, + the ceph-osd upstart script requires the id parameter to be passed along + in order to identify which running daemon should be reloaded. The follow- + ing example restarts the ceph-osd service for instance id=4: + + service_reload('ceph-osd', id=4) + + :param service_name: the name of the service to reload + :param restart_on_failure: boolean indicating whether to fallback to a + restart if the reload fails. + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems not allowing additional + parameters via the commandline (systemd). + """ + service_result = service('reload', service_name, **kwargs) + if not service_result and restart_on_failure: + service_result = service('restart', service_name, **kwargs) + return service_result + + +def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d", + **kwargs): + """Pause a system service. + + Stop it, and prevent it from starting again at boot. + + :param service_name: the name of the service to pause + :param init_dir: path to the upstart init directory + :param initd_dir: path to the sysv init directory + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for init systems which do not support + key=value arguments via the commandline. + """ + stopped = True + if service_running(service_name, **kwargs): + stopped = service_stop(service_name, **kwargs) + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) + sysv_file = os.path.join(initd_dir, service_name) + if init_is_systemd(): + service('disable', service_name) + service('mask', service_name) + elif os.path.exists(upstart_file): + override_path = os.path.join( + init_dir, '{}.override'.format(service_name)) + with open(override_path, 'w') as fh: + fh.write("manual\n") + elif os.path.exists(sysv_file): + subprocess.check_call(["update-rc.d", service_name, "disable"]) + else: + raise ValueError( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( + service_name, upstart_file, sysv_file)) + return stopped + + +def service_resume(service_name, init_dir="/etc/init", + initd_dir="/etc/init.d", **kwargs): + """Resume a system service. + + Reenable starting again at boot. Start the service. + + :param service_name: the name of the service to resume + :param init_dir: the path to the init dir + :param initd dir: the path to the initd dir + :param **kwargs: additional parameters to pass to the init system when + managing services. These will be passed as key=value + parameters to the init system's commandline. kwargs + are ignored for systemd enabled systems. + """ + upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) + sysv_file = os.path.join(initd_dir, service_name) + if init_is_systemd(): + service('unmask', service_name) + service('enable', service_name) + elif os.path.exists(upstart_file): + override_path = os.path.join( + init_dir, '{}.override'.format(service_name)) + if os.path.exists(override_path): + os.unlink(override_path) + elif os.path.exists(sysv_file): + subprocess.check_call(["update-rc.d", service_name, "enable"]) + else: + raise ValueError( + "Unable to detect {0} as SystemD, Upstart {1} or" + " SysV {2}".format( + service_name, upstart_file, sysv_file)) + started = service_running(service_name, **kwargs) + + if not started: + started = service_start(service_name, **kwargs) + return started + + +def service(action, service_name, **kwargs): + """Control a system service. + + :param action: the action to take on the service + :param service_name: the name of the service to perform th action on + :param **kwargs: additional params to be passed to the service command in + the form of key=value. + """ + if init_is_systemd(): + cmd = ['systemctl', action, service_name] + else: + cmd = ['service', service_name, action] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) + return subprocess.call(cmd) == 0 + + +_UPSTART_CONF = "/etc/init/{}.conf" +_INIT_D_CONF = "/etc/init.d/{}" + + +def service_running(service_name, **kwargs): + """Determine whether a system service is running. + + :param service_name: the name of the service + :param **kwargs: additional args to pass to the service command. This is + used to pass additional key=value arguments to the + service command line for managing specific instance + units (e.g. service ceph-osd status id=2). The kwargs + are ignored in systemd services. + """ + if init_is_systemd(): + return service('is-active', service_name) + else: + if os.path.exists(_UPSTART_CONF.format(service_name)): + try: + cmd = ['status', service_name] + for key, value in six.iteritems(kwargs): + parameter = '%s=%s' % (key, value) + cmd.append(parameter) + output = subprocess.check_output(cmd, + stderr=subprocess.STDOUT).decode('UTF-8') + except subprocess.CalledProcessError: + return False + else: + # This works for upstart scripts where the 'service' command + # returns a consistent string to represent running + # 'start/running' + if ("start/running" in output or + "is running" in output or + "up and running" in output): + return True + elif os.path.exists(_INIT_D_CONF.format(service_name)): + # Check System V scripts init script return codes + return service('status', service_name) + return False + + +SYSTEMD_SYSTEM = '/run/systemd/system' + + +def init_is_systemd(): + """Return True if the host system uses systemd, False otherwise.""" + if lsb_release()['DISTRIB_CODENAME'] == 'trusty': + return False + return os.path.isdir(SYSTEMD_SYSTEM) + + +def adduser(username, password=None, shell='/bin/bash', + system_user=False, primary_group=None, + secondary_groups=None, uid=None, home_dir=None): + """Add a user to the system. + + Will log but otherwise succeed if the user already exists. + + :param str username: Username to create + :param str password: Password for user; if ``None``, create a system user + :param str shell: The default shell for the user + :param bool system_user: Whether to create a login or system user + :param str primary_group: Primary group for user; defaults to username + :param list secondary_groups: Optional list of additional groups + :param int uid: UID for user being created + :param str home_dir: Home directory for user + + :returns: The password database entry struct, as returned by `pwd.getpwnam` + """ + try: + user_info = pwd.getpwnam(username) + log('user {0} already exists!'.format(username)) + if uid: + user_info = pwd.getpwuid(int(uid)) + log('user with uid {0} already exists!'.format(uid)) + except KeyError: + log('creating user {0}'.format(username)) + cmd = ['useradd'] + if uid: + cmd.extend(['--uid', str(uid)]) + if home_dir: + cmd.extend(['--home', str(home_dir)]) + if system_user or password is None: + cmd.append('--system') + else: + cmd.extend([ + '--create-home', + '--shell', shell, + '--password', password, + ]) + if not primary_group: + try: + grp.getgrnam(username) + primary_group = username # avoid "group exists" error + except KeyError: + pass + if primary_group: + cmd.extend(['-g', primary_group]) + if secondary_groups: + cmd.extend(['-G', ','.join(secondary_groups)]) + cmd.append(username) + subprocess.check_call(cmd) + user_info = pwd.getpwnam(username) + return user_info + + +def user_exists(username): + """Check if a user exists""" + try: + pwd.getpwnam(username) + user_exists = True + except KeyError: + user_exists = False + return user_exists + + +def uid_exists(uid): + """Check if a uid exists""" + try: + pwd.getpwuid(uid) + uid_exists = True + except KeyError: + uid_exists = False + return uid_exists + + +def group_exists(groupname): + """Check if a group exists""" + try: + grp.getgrnam(groupname) + group_exists = True + except KeyError: + group_exists = False + return group_exists + + +def gid_exists(gid): + """Check if a gid exists""" + try: + grp.getgrgid(gid) + gid_exists = True + except KeyError: + gid_exists = False + return gid_exists + + +def add_group(group_name, system_group=False, gid=None): + """Add a group to the system + + Will log but otherwise succeed if the group already exists. + + :param str group_name: group to create + :param bool system_group: Create system group + :param int gid: GID for user being created + + :returns: The password database entry struct, as returned by `grp.getgrnam` + """ + try: + group_info = grp.getgrnam(group_name) + log('group {0} already exists!'.format(group_name)) + if gid: + group_info = grp.getgrgid(gid) + log('group with gid {0} already exists!'.format(gid)) + except KeyError: + log('creating group {0}'.format(group_name)) + add_new_group(group_name, system_group, gid) + group_info = grp.getgrnam(group_name) + return group_info + + +def add_user_to_group(username, group): + """Add a user to a group""" + cmd = ['gpasswd', '-a', username, group] + log("Adding user {} to group {}".format(username, group)) + subprocess.check_call(cmd) + + +def rsync(from_path, to_path, flags='-r', options=None, timeout=None): + """Replicate the contents of a path""" + options = options or ['--delete', '--executability'] + cmd = ['/usr/bin/rsync', flags] + if timeout: + cmd = ['timeout', str(timeout)] + cmd + cmd.extend(options) + cmd.append(from_path) + cmd.append(to_path) + log(" ".join(cmd)) + return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip() + + +def symlink(source, destination): + """Create a symbolic link""" + log("Symlinking {} as {}".format(source, destination)) + cmd = [ + 'ln', + '-sf', + source, + destination, + ] + subprocess.check_call(cmd) + + +def mkdir(path, owner='root', group='root', perms=0o555, force=False): + """Create a directory""" + log("Making dir {} {}:{} {:o}".format(path, owner, group, + perms)) + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + realpath = os.path.abspath(path) + path_exists = os.path.exists(realpath) + if path_exists and force: + if not os.path.isdir(realpath): + log("Removing non-directory file {} prior to mkdir()".format(path)) + os.unlink(realpath) + os.makedirs(realpath, perms) + elif not path_exists: + os.makedirs(realpath, perms) + os.chown(realpath, uid, gid) + os.chmod(realpath, perms) + + +def write_file(path, content, owner='root', group='root', perms=0o444): + """Create or overwrite a file with the contents of a byte string.""" + log("Writing file {} {}:{} {:o}".format(path, owner, group, perms)) + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + with open(path, 'wb') as target: + os.fchown(target.fileno(), uid, gid) + os.fchmod(target.fileno(), perms) + target.write(content) + + +def fstab_remove(mp): + """Remove the given mountpoint entry from /etc/fstab""" + return Fstab.remove_by_mountpoint(mp) + + +def fstab_add(dev, mp, fs, options=None): + """Adds the given device entry to the /etc/fstab file""" + return Fstab.add(dev, mp, fs, options=options) + + +def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"): + """Mount a filesystem at a particular mountpoint""" + cmd_args = ['mount'] + if options is not None: + cmd_args.extend(['-o', options]) + cmd_args.extend([device, mountpoint]) + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output)) + return False + + if persist: + return fstab_add(device, mountpoint, filesystem, options=options) + return True + + +def umount(mountpoint, persist=False): + """Unmount a filesystem""" + cmd_args = ['umount', mountpoint] + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error unmounting {}\n{}'.format(mountpoint, e.output)) + return False + + if persist: + return fstab_remove(mountpoint) + return True + + +def mounts(): + """Get a list of all mounted volumes as [[mountpoint,device],[...]]""" + with open('/proc/mounts') as f: + # [['/mount/point','/dev/path'],[...]] + system_mounts = [m[1::-1] for m in [l.strip().split() + for l in f.readlines()]] + return system_mounts + + +def fstab_mount(mountpoint): + """Mount filesystem using fstab""" + cmd_args = ['mount', mountpoint] + try: + subprocess.check_output(cmd_args) + except subprocess.CalledProcessError as e: + log('Error unmounting {}\n{}'.format(mountpoint, e.output)) + return False + return True + + +def file_hash(path, hash_type='md5'): + """Generate a hash checksum of the contents of 'path' or None if not found. + + :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ + if os.path.exists(path): + h = getattr(hashlib, hash_type)() + with open(path, 'rb') as source: + h.update(source.read()) + return h.hexdigest() + else: + return None + + +def path_hash(path): + """Generate a hash checksum of all files matching 'path'. Standard + wildcards like '*' and '?' are supported, see documentation for the 'glob' + module for more information. + + :return: dict: A { filename: hash } dictionary for all matched files. + Empty if none found. + """ + return { + filename: file_hash(filename) + for filename in glob.iglob(path) + } + + +def check_hash(path, checksum, hash_type='md5'): + """Validate a file using a cryptographic checksum. + + :param str checksum: Value of the checksum used to validate the file. + :param str hash_type: Hash algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + :raises ChecksumError: If the file fails the checksum + + """ + actual_checksum = file_hash(path, hash_type) + if checksum != actual_checksum: + raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum)) + + +class ChecksumError(ValueError): + """A class derived from Value error to indicate the checksum failed.""" + pass + + +def restart_on_change(restart_map, stopstart=False, restart_functions=None): + """Restart services based on configuration files changing + + This function is used a decorator, for example:: + + @restart_on_change({ + '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ] + '/etc/apache/sites-enabled/*': [ 'apache2' ] + }) + def config_changed(): + pass # your code here + + In this example, the cinder-api and cinder-volume services + would be restarted if /etc/ceph/ceph.conf is changed by the + ceph_client_changed function. The apache2 service would be + restarted if any file matching the pattern got changed, created + or removed. Standard wildcards are supported, see documentation + for the 'glob' module for more information. + + @param restart_map: {path_file_name: [service_name, ...] + @param stopstart: DEFAULT false; whether to stop, start OR restart + @param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + @returns result from decorated function + """ + def wrap(f): + @functools.wraps(f) + def wrapped_f(*args, **kwargs): + return restart_on_change_helper( + (lambda: f(*args, **kwargs)), restart_map, stopstart, + restart_functions) + return wrapped_f + return wrap + + +def restart_on_change_helper(lambda_f, restart_map, stopstart=False, + restart_functions=None): + """Helper function to perform the restart_on_change function. + + This is provided for decorators to restart services if files described + in the restart_map have changed after an invocation of lambda_f(). + + @param lambda_f: function to call. + @param restart_map: {file: [service, ...]} + @param stopstart: whether to stop, start or restart a service + @param restart_functions: nonstandard functions to use to restart services + {svc: func, ...} + @returns result of lambda_f() + """ + if restart_functions is None: + restart_functions = {} + checksums = {path: path_hash(path) for path in restart_map} + r = lambda_f() + # create a list of lists of the services to restart + restarts = [restart_map[path] + for path in restart_map + if path_hash(path) != checksums[path]] + # create a flat list of ordered services without duplicates from lists + services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts))) + if services_list: + actions = ('stop', 'start') if stopstart else ('restart',) + for service_name in services_list: + if service_name in restart_functions: + restart_functions[service_name](service_name) + else: + for action in actions: + service(action, service_name) + return r + + +def pwgen(length=None): + """Generate a random pasword.""" + if length is None: + # A random length is ok to use a weak PRNG + length = random.choice(range(35, 45)) + alphanumeric_chars = [ + l for l in (string.ascii_letters + string.digits) + if l not in 'l0QD1vAEIOUaeiou'] + # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the + # actual password + random_generator = random.SystemRandom() + random_chars = [ + random_generator.choice(alphanumeric_chars) for _ in range(length)] + return(''.join(random_chars)) + + +def is_phy_iface(interface): + """Returns True if interface is not virtual, otherwise False.""" + if interface: + sys_net = '/sys/class/net' + if os.path.isdir(sys_net): + for iface in glob.glob(os.path.join(sys_net, '*')): + if '/virtual/' in os.path.realpath(iface): + continue + + if interface == os.path.basename(iface): + return True + + return False + + +def get_bond_master(interface): + """Returns bond master if interface is bond slave otherwise None. + + NOTE: the provided interface is expected to be physical + """ + if interface: + iface_path = '/sys/class/net/%s' % (interface) + if os.path.exists(iface_path): + if '/virtual/' in os.path.realpath(iface_path): + return None + + master = os.path.join(iface_path, 'master') + if os.path.exists(master): + master = os.path.realpath(master) + # make sure it is a bond master + if os.path.exists(os.path.join(master, 'bonding')): + return os.path.basename(master) + + return None + + +def list_nics(nic_type=None): + """Return a list of nics of given type(s)""" + if isinstance(nic_type, six.string_types): + int_types = [nic_type] + else: + int_types = nic_type + + interfaces = [] + if nic_type: + for int_type in int_types: + cmd = ['ip', 'addr', 'show', 'label', int_type + '*'] + ip_output = subprocess.check_output(cmd).decode('UTF-8') + ip_output = ip_output.split('\n') + ip_output = (line for line in ip_output if line) + for line in ip_output: + if line.split()[1].startswith(int_type): + matched = re.search('.*: (' + int_type + + r'[0-9]+\.[0-9]+)@.*', line) + if matched: + iface = matched.groups()[0] + else: + iface = line.split()[1].replace(":", "") + + if iface not in interfaces: + interfaces.append(iface) + else: + cmd = ['ip', 'a'] + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') + ip_output = (line.strip() for line in ip_output if line) + + key = re.compile('^[0-9]+:\s+(.+):') + for line in ip_output: + matched = re.search(key, line) + if matched: + iface = matched.group(1) + iface = iface.partition("@")[0] + if iface not in interfaces: + interfaces.append(iface) + + return interfaces + + +def set_nic_mtu(nic, mtu): + """Set the Maximum Transmission Unit (MTU) on a network interface.""" + cmd = ['ip', 'link', 'set', nic, 'mtu', mtu] + subprocess.check_call(cmd) + + +def get_nic_mtu(nic): + """Return the Maximum Transmission Unit (MTU) for a network interface.""" + cmd = ['ip', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n') + mtu = "" + for line in ip_output: + words = line.split() + if 'mtu' in words: + mtu = words[words.index("mtu") + 1] + return mtu + + +def get_nic_hwaddr(nic): + """Return the Media Access Control (MAC) for a network interface.""" + cmd = ['ip', '-o', '-0', 'addr', 'show', nic] + ip_output = subprocess.check_output(cmd).decode('UTF-8') + hwaddr = "" + words = ip_output.split() + if 'link/ether' in words: + hwaddr = words[words.index('link/ether') + 1] + return hwaddr + + +@contextmanager +def chdir(directory): + """Change the current working directory to a different directory for a code + block and return the previous directory after the block exits. Useful to + run commands from a specificed directory. + + :param str directory: The directory path to change to for this context. + """ + cur = os.getcwd() + try: + yield os.chdir(directory) + finally: + os.chdir(cur) + + +def chownr(path, owner, group, follow_links=True, chowntopdir=False): + """Recursively change user and group ownership of files and directories + in given path. Doesn't chown path itself by default, only its children. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + :param bool follow_links: Also follow and chown links if True + :param bool chowntopdir: Also chown path itself if True + """ + uid = pwd.getpwnam(owner).pw_uid + gid = grp.getgrnam(group).gr_gid + if follow_links: + chown = os.chown + else: + chown = os.lchown + + if chowntopdir: + broken_symlink = os.path.lexists(path) and not os.path.exists(path) + if not broken_symlink: + chown(path, uid, gid) + for root, dirs, files in os.walk(path, followlinks=follow_links): + for name in dirs + files: + full = os.path.join(root, name) + broken_symlink = os.path.lexists(full) and not os.path.exists(full) + if not broken_symlink: + chown(full, uid, gid) + + +def lchownr(path, owner, group): + """Recursively change user and group ownership of files and directories + in a given path, not following symbolic links. See the documentation for + 'os.lchown' for more information. + + :param str path: The string path to start changing ownership. + :param str owner: The owner string to use when looking up the uid. + :param str group: The group string to use when looking up the gid. + """ + chownr(path, owner, group, follow_links=False) + + +def owner(path): + """Returns a tuple containing the username & groupname owning the path. + + :param str path: the string path to retrieve the ownership + :return tuple(str, str): A (username, groupname) tuple containing the + name of the user and group owning the path. + :raises OSError: if the specified path does not exist + """ + stat = os.stat(path) + username = pwd.getpwuid(stat.st_uid)[0] + groupname = grp.getgrgid(stat.st_gid)[0] + return username, groupname + + +def get_total_ram(): + """The total amount of system RAM in bytes. + + This is what is reported by the OS, and may be overcommitted when + there are multiple containers hosted on the same machine. + """ + with open('/proc/meminfo', 'r') as f: + for line in f.readlines(): + if line: + key, value, unit = line.split() + if key == 'MemTotal:': + assert unit == 'kB', 'Unknown unit' + return int(value) * 1024 # Classic, not KiB. + raise NotImplementedError() + + +UPSTART_CONTAINER_TYPE = '/run/container_type' + + +def is_container(): + """Determine whether unit is running in a container + + @return: boolean indicating if unit is in a container + """ + if init_is_systemd(): + # Detect using systemd-detect-virt + return subprocess.call(['systemd-detect-virt', + '--container']) == 0 + else: + # Detect using upstart container file marker + return os.path.exists(UPSTART_CONTAINER_TYPE) + + +def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH): + with open(updatedb_path, 'r+') as f_id: + updatedb_text = f_id.read() + output = updatedb(updatedb_text, path) + f_id.seek(0) + f_id.write(output) + f_id.truncate() + + +def updatedb(updatedb_text, new_path): + lines = [line for line in updatedb_text.split("\n")] + for i, line in enumerate(lines): + if line.startswith("PRUNEPATHS="): + paths_line = line.split("=")[1].replace('"', '') + paths = paths_line.split(" ") + if new_path not in paths: + paths.append(new_path) + lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths)) + output = "\n".join(lines) + return output diff --git a/contrail-analytics/hooks/charmhelpers/core/host_factory/__init__.py b/contrail-analytics/hooks/charmhelpers/core/host_factory/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/host_factory/__init__.py diff --git a/contrail-analytics/hooks/charmhelpers/core/host_factory/centos.py b/contrail-analytics/hooks/charmhelpers/core/host_factory/centos.py new file mode 100644 index 0000000..7781a39 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/host_factory/centos.py @@ -0,0 +1,72 @@ +import subprocess +import yum +import os + +from charmhelpers.core.strutils import BasicStringComparator + + +class CompareHostReleases(BasicStringComparator): + """Provide comparisons of Host releases. + + Use in the form of + + if CompareHostReleases(release) > 'trusty': + # do something with mitaka + """ + + def __init__(self, item): + raise NotImplementedError( + "CompareHostReleases() is not implemented for CentOS") + + +def service_available(service_name): + # """Determine whether a system service is available.""" + if os.path.isdir('/run/systemd/system'): + cmd = ['systemctl', 'is-enabled', service_name] + else: + cmd = ['service', service_name, 'is-enabled'] + return subprocess.call(cmd) == 0 + + +def add_new_group(group_name, system_group=False, gid=None): + cmd = ['groupadd'] + if gid: + cmd.extend(['--gid', str(gid)]) + if system_group: + cmd.append('-r') + cmd.append(group_name) + subprocess.check_call(cmd) + + +def lsb_release(): + """Return /etc/os-release in a dict.""" + d = {} + with open('/etc/os-release', 'r') as lsb: + for l in lsb: + s = l.split('=') + if len(s) != 2: + continue + d[s[0].strip()] = s[1].strip() + return d + + +def cmp_pkgrevno(package, revno, pkgcache=None): + """Compare supplied revno with the revno of the installed package. + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + + This function imports YumBase function if the pkgcache argument + is None. + """ + if not pkgcache: + y = yum.YumBase() + packages = y.doPackageLists() + pkgcache = {i.Name: i.version for i in packages['installed']} + pkg = pkgcache[package] + if pkg > revno: + return 1 + if pkg < revno: + return -1 + return 0 diff --git a/contrail-analytics/hooks/charmhelpers/core/host_factory/ubuntu.py b/contrail-analytics/hooks/charmhelpers/core/host_factory/ubuntu.py new file mode 100644 index 0000000..d8dc378 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/host_factory/ubuntu.py @@ -0,0 +1,89 @@ +import subprocess + +from charmhelpers.core.strutils import BasicStringComparator + + +UBUNTU_RELEASES = ( + 'lucid', + 'maverick', + 'natty', + 'oneiric', + 'precise', + 'quantal', + 'raring', + 'saucy', + 'trusty', + 'utopic', + 'vivid', + 'wily', + 'xenial', + 'yakkety', + 'zesty', + 'artful', +) + + +class CompareHostReleases(BasicStringComparator): + """Provide comparisons of Ubuntu releases. + + Use in the form of + + if CompareHostReleases(release) > 'trusty': + # do something with mitaka + """ + _list = UBUNTU_RELEASES + + +def service_available(service_name): + """Determine whether a system service is available""" + try: + subprocess.check_output( + ['service', service_name, 'status'], + stderr=subprocess.STDOUT).decode('UTF-8') + except subprocess.CalledProcessError as e: + return b'unrecognized service' not in e.output + else: + return True + + +def add_new_group(group_name, system_group=False, gid=None): + cmd = ['addgroup'] + if gid: + cmd.extend(['--gid', str(gid)]) + if system_group: + cmd.append('--system') + else: + cmd.extend([ + '--group', + ]) + cmd.append(group_name) + subprocess.check_call(cmd) + + +def lsb_release(): + """Return /etc/lsb-release in a dict""" + d = {} + with open('/etc/lsb-release', 'r') as lsb: + for l in lsb: + k, v = l.split('=') + d[k.strip()] = v.strip() + return d + + +def cmp_pkgrevno(package, revno, pkgcache=None): + """Compare supplied revno with the revno of the installed package. + + * 1 => Installed revno is greater than supplied arg + * 0 => Installed revno is the same as supplied arg + * -1 => Installed revno is less than supplied arg + + This function imports apt_cache function from charmhelpers.fetch if + the pkgcache argument is None. Be sure to add charmhelpers.fetch if + you call this function, or pass an apt_pkg.Cache() instance. + """ + import apt_pkg + if not pkgcache: + from charmhelpers.fetch import apt_cache + pkgcache = apt_cache() + pkg = pkgcache[package] + return apt_pkg.version_compare(pkg.current_ver.ver_str, revno) diff --git a/contrail-analytics/hooks/charmhelpers/core/hugepage.py b/contrail-analytics/hooks/charmhelpers/core/hugepage.py new file mode 100644 index 0000000..54b5b5e --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/hugepage.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +from charmhelpers.core import fstab +from charmhelpers.core import sysctl +from charmhelpers.core.host import ( + add_group, + add_user_to_group, + fstab_mount, + mkdir, +) +from charmhelpers.core.strutils import bytes_from_string +from subprocess import check_output + + +def hugepage_support(user, group='hugetlb', nr_hugepages=256, + max_map_count=65536, mnt_point='/run/hugepages/kvm', + pagesize='2MB', mount=True, set_shmmax=False): + """Enable hugepages on system. + + Args: + user (str) -- Username to allow access to hugepages to + group (str) -- Group name to own hugepages + nr_hugepages (int) -- Number of pages to reserve + max_map_count (int) -- Number of Virtual Memory Areas a process can own + mnt_point (str) -- Directory to mount hugepages on + pagesize (str) -- Size of hugepages + mount (bool) -- Whether to Mount hugepages + """ + group_info = add_group(group) + gid = group_info.gr_gid + add_user_to_group(user, group) + if max_map_count < 2 * nr_hugepages: + max_map_count = 2 * nr_hugepages + sysctl_settings = { + 'vm.nr_hugepages': nr_hugepages, + 'vm.max_map_count': max_map_count, + 'vm.hugetlb_shm_group': gid, + } + if set_shmmax: + shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax'])) + shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages + if shmmax_minsize > shmmax_current: + sysctl_settings['kernel.shmmax'] = shmmax_minsize + sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf') + mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False) + lfstab = fstab.Fstab() + fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point) + if fstab_entry: + lfstab.remove_entry(fstab_entry) + entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs', + 'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0) + lfstab.add_entry(entry) + if mount: + fstab_mount(mnt_point) diff --git a/contrail-analytics/hooks/charmhelpers/core/kernel.py b/contrail-analytics/hooks/charmhelpers/core/kernel.py new file mode 100644 index 0000000..2d40452 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/kernel.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import subprocess + +from charmhelpers.osplatform import get_platform +from charmhelpers.core.hookenv import ( + log, + INFO +) + +__platform__ = get_platform() +if __platform__ == "ubuntu": + from charmhelpers.core.kernel_factory.ubuntu import ( + persistent_modprobe, + update_initramfs, + ) # flake8: noqa -- ignore F401 for this import +elif __platform__ == "centos": + from charmhelpers.core.kernel_factory.centos import ( + persistent_modprobe, + update_initramfs, + ) # flake8: noqa -- ignore F401 for this import + +__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>" + + +def modprobe(module, persist=True): + """Load a kernel module and configure for auto-load on reboot.""" + cmd = ['modprobe', module] + + log('Loading kernel module %s' % module, level=INFO) + + subprocess.check_call(cmd) + if persist: + persistent_modprobe(module) + + +def rmmod(module, force=False): + """Remove a module from the linux kernel""" + cmd = ['rmmod'] + if force: + cmd.append('-f') + cmd.append(module) + log('Removing kernel module %s' % module, level=INFO) + return subprocess.check_call(cmd) + + +def lsmod(): + """Shows what kernel modules are currently loaded""" + return subprocess.check_output(['lsmod'], + universal_newlines=True) + + +def is_module_loaded(module): + """Checks if a kernel module is already loaded""" + matches = re.findall('^%s[ ]+' % module, lsmod(), re.M) + return len(matches) > 0 diff --git a/contrail-analytics/hooks/charmhelpers/core/kernel_factory/__init__.py b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/__init__.py diff --git a/contrail-analytics/hooks/charmhelpers/core/kernel_factory/centos.py b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/centos.py new file mode 100644 index 0000000..1c402c1 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/centos.py @@ -0,0 +1,17 @@ +import subprocess +import os + + +def persistent_modprobe(module): + """Load a kernel module and configure for auto-load on reboot.""" + if not os.path.exists('/etc/rc.modules'): + open('/etc/rc.modules', 'a') + os.chmod('/etc/rc.modules', 111) + with open('/etc/rc.modules', 'r+') as modules: + if module not in modules.read(): + modules.write('modprobe %s\n' % module) + + +def update_initramfs(version='all'): + """Updates an initramfs image.""" + return subprocess.check_call(["dracut", "-f", version]) diff --git a/contrail-analytics/hooks/charmhelpers/core/kernel_factory/ubuntu.py b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/ubuntu.py new file mode 100644 index 0000000..3de372f --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/kernel_factory/ubuntu.py @@ -0,0 +1,13 @@ +import subprocess + + +def persistent_modprobe(module): + """Load a kernel module and configure for auto-load on reboot.""" + with open('/etc/modules', 'r+') as modules: + if module not in modules.read(): + modules.write(module + "\n") + + +def update_initramfs(version='all'): + """Updates an initramfs image.""" + return subprocess.check_call(["update-initramfs", "-k", version, "-u"]) diff --git a/contrail-analytics/hooks/charmhelpers/core/services/__init__.py b/contrail-analytics/hooks/charmhelpers/core/services/__init__.py new file mode 100644 index 0000000..61fd074 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/services/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .base import * # NOQA +from .helpers import * # NOQA diff --git a/contrail-analytics/hooks/charmhelpers/core/services/base.py b/contrail-analytics/hooks/charmhelpers/core/services/base.py new file mode 100644 index 0000000..ca9dc99 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/services/base.py @@ -0,0 +1,351 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +from inspect import getargspec +from collections import Iterable, OrderedDict + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +__all__ = ['ServiceManager', 'ManagerCallback', + 'PortManagerCallback', 'open_ports', 'close_ports', 'manage_ports', + 'service_restart', 'service_stop'] + + +class ServiceManager(object): + def __init__(self, services=None): + """ + Register a list of services, given their definitions. + + Service definitions are dicts in the following formats (all keys except + 'service' are optional):: + + { + "service": <service name>, + "required_data": <list of required data contexts>, + "provided_data": <list of provided data contexts>, + "data_ready": <one or more callbacks>, + "data_lost": <one or more callbacks>, + "start": <one or more callbacks>, + "stop": <one or more callbacks>, + "ports": <list of ports to manage>, + } + + The 'required_data' list should contain dicts of required data (or + dependency managers that act like dicts and know how to collect the data). + Only when all items in the 'required_data' list are populated are the list + of 'data_ready' and 'start' callbacks executed. See `is_ready()` for more + information. + + The 'provided_data' list should contain relation data providers, most likely + a subclass of :class:`charmhelpers.core.services.helpers.RelationContext`, + that will indicate a set of data to set on a given relation. + + The 'data_ready' value should be either a single callback, or a list of + callbacks, to be called when all items in 'required_data' pass `is_ready()`. + Each callback will be called with the service name as the only parameter. + After all of the 'data_ready' callbacks are called, the 'start' callbacks + are fired. + + The 'data_lost' value should be either a single callback, or a list of + callbacks, to be called when a 'required_data' item no longer passes + `is_ready()`. Each callback will be called with the service name as the + only parameter. After all of the 'data_lost' callbacks are called, + the 'stop' callbacks are fired. + + The 'start' value should be either a single callback, or a list of + callbacks, to be called when starting the service, after the 'data_ready' + callbacks are complete. Each callback will be called with the service + name as the only parameter. This defaults to + `[host.service_start, services.open_ports]`. + + The 'stop' value should be either a single callback, or a list of + callbacks, to be called when stopping the service. If the service is + being stopped because it no longer has all of its 'required_data', this + will be called after all of the 'data_lost' callbacks are complete. + Each callback will be called with the service name as the only parameter. + This defaults to `[services.close_ports, host.service_stop]`. + + The 'ports' value should be a list of ports to manage. The default + 'start' handler will open the ports after the service is started, + and the default 'stop' handler will close the ports prior to stopping + the service. + + + Examples: + + The following registers an Upstart service called bingod that depends on + a mongodb relation and which runs a custom `db_migrate` function prior to + restarting the service, and a Runit service called spadesd:: + + manager = services.ServiceManager([ + { + 'service': 'bingod', + 'ports': [80, 443], + 'required_data': [MongoRelation(), config(), {'my': 'data'}], + 'data_ready': [ + services.template(source='bingod.conf'), + services.template(source='bingod.ini', + target='/etc/bingod.ini', + owner='bingo', perms=0400), + ], + }, + { + 'service': 'spadesd', + 'data_ready': services.template(source='spadesd_run.j2', + target='/etc/sv/spadesd/run', + perms=0555), + 'start': runit_start, + 'stop': runit_stop, + }, + ]) + manager.manage() + """ + self._ready_file = os.path.join(hookenv.charm_dir(), 'READY-SERVICES.json') + self._ready = None + self.services = OrderedDict() + for service in services or []: + service_name = service['service'] + self.services[service_name] = service + + def manage(self): + """ + Handle the current hook by doing The Right Thing with the registered services. + """ + hookenv._run_atstart() + try: + hook_name = hookenv.hook_name() + if hook_name == 'stop': + self.stop_services() + else: + self.reconfigure_services() + self.provide_data() + except SystemExit as x: + if x.code is None or x.code == 0: + hookenv._run_atexit() + hookenv._run_atexit() + + def provide_data(self): + """ + Set the relation data for each provider in the ``provided_data`` list. + + A provider must have a `name` attribute, which indicates which relation + to set data on, and a `provide_data()` method, which returns a dict of + data to set. + + The `provide_data()` method can optionally accept two parameters: + + * ``remote_service`` The name of the remote service that the data will + be provided to. The `provide_data()` method will be called once + for each connected service (not unit). This allows the method to + tailor its data to the given service. + * ``service_ready`` Whether or not the service definition had all of + its requirements met, and thus the ``data_ready`` callbacks run. + + Note that the ``provided_data`` methods are now called **after** the + ``data_ready`` callbacks are run. This gives the ``data_ready`` callbacks + a chance to generate any data necessary for the providing to the remote + services. + """ + for service_name, service in self.services.items(): + service_ready = self.is_ready(service_name) + for provider in service.get('provided_data', []): + for relid in hookenv.relation_ids(provider.name): + units = hookenv.related_units(relid) + if not units: + continue + remote_service = units[0].split('/')[0] + argspec = getargspec(provider.provide_data) + if len(argspec.args) > 1: + data = provider.provide_data(remote_service, service_ready) + else: + data = provider.provide_data() + if data: + hookenv.relation_set(relid, data) + + def reconfigure_services(self, *service_names): + """ + Update all files for one or more registered services, and, + if ready, optionally restart them. + + If no service names are given, reconfigures all registered services. + """ + for service_name in service_names or self.services.keys(): + if self.is_ready(service_name): + self.fire_event('data_ready', service_name) + self.fire_event('start', service_name, default=[ + service_restart, + manage_ports]) + self.save_ready(service_name) + else: + if self.was_ready(service_name): + self.fire_event('data_lost', service_name) + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + self.save_lost(service_name) + + def stop_services(self, *service_names): + """ + Stop one or more registered services, by name. + + If no service names are given, stops all registered services. + """ + for service_name in service_names or self.services.keys(): + self.fire_event('stop', service_name, default=[ + manage_ports, + service_stop]) + + def get_service(self, service_name): + """ + Given the name of a registered service, return its service definition. + """ + service = self.services.get(service_name) + if not service: + raise KeyError('Service not registered: %s' % service_name) + return service + + def fire_event(self, event_name, service_name, default=None): + """ + Fire a data_ready, data_lost, start, or stop event on a given service. + """ + service = self.get_service(service_name) + callbacks = service.get(event_name, default) + if not callbacks: + return + if not isinstance(callbacks, Iterable): + callbacks = [callbacks] + for callback in callbacks: + if isinstance(callback, ManagerCallback): + callback(self, service_name, event_name) + else: + callback(service_name) + + def is_ready(self, service_name): + """ + Determine if a registered service is ready, by checking its 'required_data'. + + A 'required_data' item can be any mapping type, and is considered ready + if `bool(item)` evaluates as True. + """ + service = self.get_service(service_name) + reqs = service.get('required_data', []) + return all(bool(req) for req in reqs) + + def _load_ready_file(self): + if self._ready is not None: + return + if os.path.exists(self._ready_file): + with open(self._ready_file) as fp: + self._ready = set(json.load(fp)) + else: + self._ready = set() + + def _save_ready_file(self): + if self._ready is None: + return + with open(self._ready_file, 'w') as fp: + json.dump(list(self._ready), fp) + + def save_ready(self, service_name): + """ + Save an indicator that the given service is now data_ready. + """ + self._load_ready_file() + self._ready.add(service_name) + self._save_ready_file() + + def save_lost(self, service_name): + """ + Save an indicator that the given service is no longer data_ready. + """ + self._load_ready_file() + self._ready.discard(service_name) + self._save_ready_file() + + def was_ready(self, service_name): + """ + Determine if the given service was previously data_ready. + """ + self._load_ready_file() + return service_name in self._ready + + +class ManagerCallback(object): + """ + Special case of a callback that takes the `ServiceManager` instance + in addition to the service name. + + Subclasses should implement `__call__` which should accept three parameters: + + * `manager` The `ServiceManager` instance + * `service_name` The name of the service it's being triggered for + * `event_name` The name of the event that this callback is handling + """ + def __call__(self, manager, service_name, event_name): + raise NotImplementedError() + + +class PortManagerCallback(ManagerCallback): + """ + Callback class that will open or close ports, for use as either + a start or stop action. + """ + def __call__(self, manager, service_name, event_name): + service = manager.get_service(service_name) + new_ports = service.get('ports', []) + port_file = os.path.join(hookenv.charm_dir(), '.{}.ports'.format(service_name)) + if os.path.exists(port_file): + with open(port_file) as fp: + old_ports = fp.read().split(',') + for old_port in old_ports: + if bool(old_port): + old_port = int(old_port) + if old_port not in new_ports: + hookenv.close_port(old_port) + with open(port_file, 'w') as fp: + fp.write(','.join(str(port) for port in new_ports)) + for port in new_ports: + if event_name == 'start': + hookenv.open_port(port) + elif event_name == 'stop': + hookenv.close_port(port) + + +def service_stop(service_name): + """ + Wrapper around host.service_stop to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_running(service_name): + host.service_stop(service_name) + + +def service_restart(service_name): + """ + Wrapper around host.service_restart to prevent spurious "unknown service" + messages in the logs. + """ + if host.service_available(service_name): + if host.service_running(service_name): + host.service_restart(service_name) + else: + host.service_start(service_name) + + +# Convenience aliases +open_ports = close_ports = manage_ports = PortManagerCallback() diff --git a/contrail-analytics/hooks/charmhelpers/core/services/helpers.py b/contrail-analytics/hooks/charmhelpers/core/services/helpers.py new file mode 100644 index 0000000..3e6e30d --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/services/helpers.py @@ -0,0 +1,290 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml + +from charmhelpers.core import hookenv +from charmhelpers.core import host +from charmhelpers.core import templating + +from charmhelpers.core.services.base import ManagerCallback + + +__all__ = ['RelationContext', 'TemplateCallback', + 'render_template', 'template'] + + +class RelationContext(dict): + """ + Base class for a context generator that gets relation data from juju. + + Subclasses must provide the attributes `name`, which is the name of the + interface of interest, `interface`, which is the type of the interface of + interest, and `required_keys`, which is the set of keys required for the + relation to be considered complete. The data for all interfaces matching + the `name` attribute that are complete will used to populate the dictionary + values (see `get_data`, below). + + The generated context will be namespaced under the relation :attr:`name`, + to prevent potential naming conflicts. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = None + interface = None + + def __init__(self, name=None, additional_required_keys=None): + if not hasattr(self, 'required_keys'): + self.required_keys = [] + + if name is not None: + self.name = name + if additional_required_keys: + self.required_keys.extend(additional_required_keys) + self.get_data() + + def __bool__(self): + """ + Returns True if all of the required_keys are available. + """ + return self.is_ready() + + __nonzero__ = __bool__ + + def __repr__(self): + return super(RelationContext, self).__repr__() + + def is_ready(self): + """ + Returns True if all of the `required_keys` are available from any units. + """ + ready = len(self.get(self.name, [])) > 0 + if not ready: + hookenv.log('Incomplete relation: {}'.format(self.__class__.__name__), hookenv.DEBUG) + return ready + + def _is_ready(self, unit_data): + """ + Helper method that tests a set of relation data and returns True if + all of the `required_keys` are present. + """ + return set(unit_data.keys()).issuperset(set(self.required_keys)) + + def get_data(self): + """ + Retrieve the relation data for each unit involved in a relation and, + if complete, store it in a list under `self[self.name]`. This + is automatically called when the RelationContext is instantiated. + + The units are sorted lexographically first by the service ID, then by + the unit ID. Thus, if an interface has two other services, 'db:1' + and 'db:2', with 'db:1' having two units, 'wordpress/0' and 'wordpress/1', + and 'db:2' having one unit, 'mediawiki/0', all of which have a complete + set of data, the relation data for the units will be stored in the + order: 'wordpress/0', 'wordpress/1', 'mediawiki/0'. + + If you only care about a single unit on the relation, you can just + access it as `{{ interface[0]['key'] }}`. However, if you can at all + support multiple units on a relation, you should iterate over the list, + like:: + + {% for unit in interface -%} + {{ unit['key'] }}{% if not loop.last %},{% endif %} + {%- endfor %} + + Note that since all sets of relation data from all related services and + units are in a single list, if you need to know which service or unit a + set of data came from, you'll need to extend this class to preserve + that information. + """ + if not hookenv.relation_ids(self.name): + return + + ns = self.setdefault(self.name, []) + for rid in sorted(hookenv.relation_ids(self.name)): + for unit in sorted(hookenv.related_units(rid)): + reldata = hookenv.relation_get(rid=rid, unit=unit) + if self._is_ready(reldata): + ns.append(reldata) + + def provide_data(self): + """ + Return data to be relation_set for this interface. + """ + return {} + + +class MysqlRelation(RelationContext): + """ + Relation context for the `mysql` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'db' + interface = 'mysql' + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'user', 'password', 'database'] + RelationContext.__init__(self, *args, **kwargs) + + +class HttpRelation(RelationContext): + """ + Relation context for the `http` interface. + + :param str name: Override the relation :attr:`name`, since it can vary from charm to charm + :param list additional_required_keys: Extend the list of :attr:`required_keys` + """ + name = 'website' + interface = 'http' + + def __init__(self, *args, **kwargs): + self.required_keys = ['host', 'port'] + RelationContext.__init__(self, *args, **kwargs) + + def provide_data(self): + return { + 'host': hookenv.unit_get('private-address'), + 'port': 80, + } + + +class RequiredConfig(dict): + """ + Data context that loads config options with one or more mandatory options. + + Once the required options have been changed from their default values, all + config options will be available, namespaced under `config` to prevent + potential naming conflicts (for example, between a config option and a + relation property). + + :param list *args: List of options that must be changed from their default values. + """ + + def __init__(self, *args): + self.required_options = args + self['config'] = hookenv.config() + with open(os.path.join(hookenv.charm_dir(), 'config.yaml')) as fp: + self.config = yaml.load(fp).get('options', {}) + + def __bool__(self): + for option in self.required_options: + if option not in self['config']: + return False + current_value = self['config'][option] + default_value = self.config[option].get('default') + if current_value == default_value: + return False + if current_value in (None, '') and default_value in (None, ''): + return False + return True + + def __nonzero__(self): + return self.__bool__() + + +class StoredContext(dict): + """ + A data context that always returns the data that it was first created with. + + This is useful to do a one-time generation of things like passwords, that + will thereafter use the same value that was originally generated, instead + of generating a new value each time it is run. + """ + def __init__(self, file_name, config_data): + """ + If the file exists, populate `self` with the data from the file. + Otherwise, populate with the given data and persist it to the file. + """ + if os.path.exists(file_name): + self.update(self.read_context(file_name)) + else: + self.store_context(file_name, config_data) + self.update(config_data) + + def store_context(self, file_name, config_data): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'w') as file_stream: + os.fchmod(file_stream.fileno(), 0o600) + yaml.dump(config_data, file_stream) + + def read_context(self, file_name): + if not os.path.isabs(file_name): + file_name = os.path.join(hookenv.charm_dir(), file_name) + with open(file_name, 'r') as file_stream: + data = yaml.load(file_stream) + if not data: + raise OSError("%s is empty" % file_name) + return data + + +class TemplateCallback(ManagerCallback): + """ + Callback class that will render a Jinja2 template, for use as a ready + action. + + :param str source: The template source file, relative to + `$CHARM_DIR/templates` + + :param str target: The target to write the rendered template to (or None) + :param str owner: The owner of the rendered file + :param str group: The group of the rendered file + :param int perms: The permissions of the rendered file + :param partial on_change_action: functools partial to be executed when + rendered file changes + :param jinja2 loader template_loader: A jinja2 template loader + + :return str: The rendered template + """ + def __init__(self, source, target, + owner='root', group='root', perms=0o444, + on_change_action=None, template_loader=None): + self.source = source + self.target = target + self.owner = owner + self.group = group + self.perms = perms + self.on_change_action = on_change_action + self.template_loader = template_loader + + def __call__(self, manager, service_name, event_name): + pre_checksum = '' + if self.on_change_action and os.path.isfile(self.target): + pre_checksum = host.file_hash(self.target) + service = manager.get_service(service_name) + context = {'ctx': {}} + for ctx in service.get('required_data', []): + context.update(ctx) + context['ctx'].update(ctx) + + result = templating.render(self.source, self.target, context, + self.owner, self.group, self.perms, + template_loader=self.template_loader) + if self.on_change_action: + if pre_checksum == host.file_hash(self.target): + hookenv.log( + 'No change detected: {}'.format(self.target), + hookenv.DEBUG) + else: + self.on_change_action() + + return result + + +# Convenience aliases for templates +render_template = template = TemplateCallback diff --git a/contrail-analytics/hooks/charmhelpers/core/strutils.py b/contrail-analytics/hooks/charmhelpers/core/strutils.py new file mode 100644 index 0000000..685dabd --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/strutils.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import six +import re + + +def bool_from_string(value): + """Interpret string value as boolean. + + Returns True if value translates to True otherwise False. + """ + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + + value = value.strip().lower() + + if value in ['y', 'yes', 'true', 't', 'on']: + return True + elif value in ['n', 'no', 'false', 'f', 'off']: + return False + + msg = "Unable to interpret string value '%s' as boolean" % (value) + raise ValueError(msg) + + +def bytes_from_string(value): + """Interpret human readable string value as bytes. + + Returns int + """ + BYTE_POWER = { + 'K': 1, + 'KB': 1, + 'M': 2, + 'MB': 2, + 'G': 3, + 'GB': 3, + 'T': 4, + 'TB': 4, + 'P': 5, + 'PB': 5, + } + if isinstance(value, six.string_types): + value = six.text_type(value) + else: + msg = "Unable to interpret non-string value '%s' as boolean" % (value) + raise ValueError(msg) + matches = re.match("([0-9]+)([a-zA-Z]+)", value) + if not matches: + msg = "Unable to interpret string value '%s' as bytes" % (value) + raise ValueError(msg) + return int(matches.group(1)) * (1024 ** BYTE_POWER[matches.group(2)]) + + +class BasicStringComparator(object): + """Provides a class that will compare strings from an iterator type object. + Used to provide > and < comparisons on strings that may not necessarily be + alphanumerically ordered. e.g. OpenStack or Ubuntu releases AFTER the + z-wrap. + """ + + _list = None + + def __init__(self, item): + if self._list is None: + raise Exception("Must define the _list in the class definition!") + try: + self.index = self._list.index(item) + except Exception: + raise KeyError("Item '{}' is not in list '{}'" + .format(item, self._list)) + + def __eq__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index == self._list.index(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __lt__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index < self._list.index(other) + + def __ge__(self, other): + return not self.__lt__(other) + + def __gt__(self, other): + assert isinstance(other, str) or isinstance(other, self.__class__) + return self.index > self._list.index(other) + + def __le__(self, other): + return not self.__gt__(other) + + def __str__(self): + """Always give back the item at the index so it can be used in + comparisons like: + + s_mitaka = CompareOpenStack('mitaka') + s_newton = CompareOpenstack('newton') + + assert s_newton > s_mitaka + + @returns: <string> + """ + return self._list[self.index] diff --git a/contrail-analytics/hooks/charmhelpers/core/sysctl.py b/contrail-analytics/hooks/charmhelpers/core/sysctl.py new file mode 100644 index 0000000..6e413e3 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/sysctl.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from subprocess import check_call + +from charmhelpers.core.hookenv import ( + log, + DEBUG, + ERROR, +) + +__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>' + + +def create(sysctl_dict, sysctl_file): + """Creates a sysctl.conf file from a YAML associative array + + :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :type sysctl_dict: str + :param sysctl_file: path to the sysctl file to be saved + :type sysctl_file: str or unicode + :returns: None + """ + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + + with open(sysctl_file, "w") as fd: + for key, value in sysctl_dict_parsed.items(): + fd.write("{}={}\n".format(key, value)) + + log("Updating sysctl_file: %s values: %s" % (sysctl_file, sysctl_dict_parsed), + level=DEBUG) + + check_call(["sysctl", "-p", sysctl_file]) diff --git a/contrail-analytics/hooks/charmhelpers/core/templating.py b/contrail-analytics/hooks/charmhelpers/core/templating.py new file mode 100644 index 0000000..7b801a3 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/templating.py @@ -0,0 +1,84 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + +from charmhelpers.core import host +from charmhelpers.core import hookenv + + +def render(source, target, context, owner='root', group='root', + perms=0o444, templates_dir=None, encoding='UTF-8', template_loader=None): + """ + Render a template. + + The `source` path, if not absolute, is relative to the `templates_dir`. + + The `target` path should be absolute. It can also be `None`, in which + case no file will be written. + + The context should be a dict containing the values to be replaced in the + template. + + The `owner`, `group`, and `perms` options will be passed to `write_file`. + + If omitted, `templates_dir` defaults to the `templates` folder in the charm. + + The rendered template will be written to the file as well as being returned + as a string. + + Note: Using this requires python-jinja2 or python3-jinja2; if it is not + installed, calling this will attempt to use charmhelpers.fetch.apt_install + to install it. + """ + try: + from jinja2 import FileSystemLoader, Environment, exceptions + except ImportError: + try: + from charmhelpers.fetch import apt_install + except ImportError: + hookenv.log('Could not import jinja2, and could not import ' + 'charmhelpers.fetch to install it', + level=hookenv.ERROR) + raise + if sys.version_info.major == 2: + apt_install('python-jinja2', fatal=True) + else: + apt_install('python3-jinja2', fatal=True) + from jinja2 import FileSystemLoader, Environment, exceptions + + if template_loader: + template_env = Environment(loader=template_loader) + else: + if templates_dir is None: + templates_dir = os.path.join(hookenv.charm_dir(), 'templates') + template_env = Environment(loader=FileSystemLoader(templates_dir)) + try: + source = source + template = template_env.get_template(source) + except exceptions.TemplateNotFound as e: + hookenv.log('Could not load template %s from %s.' % + (source, templates_dir), + level=hookenv.ERROR) + raise e + content = template.render(context) + if target is not None: + target_dir = os.path.dirname(target) + if not os.path.exists(target_dir): + # This is a terrible default directory permission, as the file + # or its siblings will often contain secrets. + host.mkdir(os.path.dirname(target), owner, group, perms=0o755) + host.write_file(target, content.encode(encoding), owner, group, perms) + return content diff --git a/contrail-analytics/hooks/charmhelpers/core/unitdata.py b/contrail-analytics/hooks/charmhelpers/core/unitdata.py new file mode 100644 index 0000000..54ec969 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/core/unitdata.py @@ -0,0 +1,518 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Authors: +# Kapil Thangavelu <kapil.foss@gmail.com> +# +""" +Intro +----- + +A simple way to store state in units. This provides a key value +storage with support for versioned, transactional operation, +and can calculate deltas from previous values to simplify unit logic +when processing changes. + + +Hook Integration +---------------- + +There are several extant frameworks for hook execution, including + + - charmhelpers.core.hookenv.Hooks + - charmhelpers.core.services.ServiceManager + +The storage classes are framework agnostic, one simple integration is +via the HookData contextmanager. It will record the current hook +execution environment (including relation data, config data, etc.), +setup a transaction and allow easy access to the changes from +previously seen values. One consequence of the integration is the +reservation of particular keys ('rels', 'unit', 'env', 'config', +'charm_revisions') for their respective values. + +Here's a fully worked integration example using hookenv.Hooks:: + + from charmhelper.core import hookenv, unitdata + + hook_data = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # Print all changes to configuration from previously seen + # values. + for changed, (prev, cur) in hook_data.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + # Directly access all charm config as a mapping. + conf = db.getrange('config', True) + + # Directly access all relation data as a mapping + rels = db.getrange('rels', True) + + if __name__ == '__main__': + with hook_data(): + hook.execute() + + +A more basic integration is via the hook_scope context manager which simply +manages transaction scope (and records hook name, and timestamp):: + + >>> from unitdata import kv + >>> db = kv() + >>> with db.hook_scope('install'): + ... # do work, in transactional scope. + ... db.set('x', 1) + >>> db.get('x') + 1 + + +Usage +----- + +Values are automatically json de/serialized to preserve basic typing +and complex data struct capabilities (dicts, lists, ints, booleans, etc). + +Individual values can be manipulated via get/set:: + + >>> kv.set('y', True) + >>> kv.get('y') + True + + # We can set complex values (dicts, lists) as a single key. + >>> kv.set('config', {'a': 1, 'b': True'}) + + # Also supports returning dictionaries as a record which + # provides attribute access. + >>> config = kv.get('config', record=True) + >>> config.b + True + + +Groups of keys can be manipulated with update/getrange:: + + >>> kv.update({'z': 1, 'y': 2}, prefix="gui.") + >>> kv.getrange('gui.', strip=True) + {'z': 1, 'y': 2} + +When updating values, its very helpful to understand which values +have actually changed and how have they changed. The storage +provides a delta method to provide for this:: + + >>> data = {'debug': True, 'option': 2} + >>> delta = kv.delta(data, 'config.') + >>> delta.debug.previous + None + >>> delta.debug.current + True + >>> delta + {'debug': (None, True), 'option': (None, 2)} + +Note the delta method does not persist the actual change, it needs to +be explicitly saved via 'update' method:: + + >>> kv.update(data, 'config.') + +Values modified in the context of a hook scope retain historical values +associated to the hookname. + + >>> with db.hook_scope('config-changed'): + ... db.set('x', 42) + >>> db.gethistory('x') + [(1, u'x', 1, u'install', u'2015-01-21T16:49:30.038372'), + (2, u'x', 42, u'config-changed', u'2015-01-21T16:49:30.038786')] + +""" + +import collections +import contextlib +import datetime +import itertools +import json +import os +import pprint +import sqlite3 +import sys + +__author__ = 'Kapil Thangavelu <kapil.foss@gmail.com>' + + +class Storage(object): + """Simple key value database for local unit state within charms. + + Modifications are not persisted unless :meth:`flush` is called. + + To support dicts, lists, integer, floats, and booleans values + are automatically json encoded/decoded. + """ + def __init__(self, path=None): + self.db_path = path + if path is None: + if 'UNIT_STATE_DB' in os.environ: + self.db_path = os.environ['UNIT_STATE_DB'] + else: + self.db_path = os.path.join( + os.environ.get('CHARM_DIR', ''), '.unit-state.db') + self.conn = sqlite3.connect('%s' % self.db_path) + self.cursor = self.conn.cursor() + self.revision = None + self._closed = False + self._init() + + def close(self): + if self._closed: + return + self.flush(False) + self.cursor.close() + self.conn.close() + self._closed = True + + def get(self, key, default=None, record=False): + self.cursor.execute('select data from kv where key=?', [key]) + result = self.cursor.fetchone() + if not result: + return default + if record: + return Record(json.loads(result[0])) + return json.loads(result[0]) + + def getrange(self, key_prefix, strip=False): + """ + Get a range of keys starting with a common prefix as a mapping of + keys to values. + + :param str key_prefix: Common prefix among all keys + :param bool strip: Optionally strip the common prefix from the key + names in the returned dict + :return dict: A (possibly empty) dict of key-value mappings + """ + self.cursor.execute("select key, data from kv where key like ?", + ['%s%%' % key_prefix]) + result = self.cursor.fetchall() + + if not result: + return {} + if not strip: + key_prefix = '' + return dict([ + (k[len(key_prefix):], json.loads(v)) for k, v in result]) + + def update(self, mapping, prefix=""): + """ + Set the values of multiple keys at once. + + :param dict mapping: Mapping of keys to values + :param str prefix: Optional prefix to apply to all keys in `mapping` + before setting + """ + for k, v in mapping.items(): + self.set("%s%s" % (prefix, k), v) + + def unset(self, key): + """ + Remove a key from the database entirely. + """ + self.cursor.execute('delete from kv where key=?', [key]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + [key, self.revision, json.dumps('DELETED')]) + + def unsetrange(self, keys=None, prefix=""): + """ + Remove a range of keys starting with a common prefix, from the database + entirely. + + :param list keys: List of keys to remove. + :param str prefix: Optional prefix to apply to all keys in ``keys`` + before removing. + """ + if keys is not None: + keys = ['%s%s' % (prefix, key) for key in keys] + self.cursor.execute('delete from kv where key in (%s)' % ','.join(['?'] * len(keys)), keys) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values %s' % ','.join(['(?, ?, ?)'] * len(keys)), + list(itertools.chain.from_iterable((key, self.revision, json.dumps('DELETED')) for key in keys))) + else: + self.cursor.execute('delete from kv where key like ?', + ['%s%%' % prefix]) + if self.revision and self.cursor.rowcount: + self.cursor.execute( + 'insert into kv_revisions values (?, ?, ?)', + ['%s%%' % prefix, self.revision, json.dumps('DELETED')]) + + def set(self, key, value): + """ + Set a value in the database. + + :param str key: Key to set the value for + :param value: Any JSON-serializable value to be set + """ + serialized = json.dumps(value) + + self.cursor.execute('select data from kv where key=?', [key]) + exists = self.cursor.fetchone() + + # Skip mutations to the same value + if exists: + if exists[0] == serialized: + return value + + if not exists: + self.cursor.execute( + 'insert into kv (key, data) values (?, ?)', + (key, serialized)) + else: + self.cursor.execute(''' + update kv + set data = ? + where key = ?''', [serialized, key]) + + # Save + if not self.revision: + return value + + self.cursor.execute( + 'select 1 from kv_revisions where key=? and revision=?', + [key, self.revision]) + exists = self.cursor.fetchone() + + if not exists: + self.cursor.execute( + '''insert into kv_revisions ( + revision, key, data) values (?, ?, ?)''', + (self.revision, key, serialized)) + else: + self.cursor.execute( + ''' + update kv_revisions + set data = ? + where key = ? + and revision = ?''', + [serialized, key, self.revision]) + + return value + + def delta(self, mapping, prefix): + """ + return a delta containing values that have changed. + """ + previous = self.getrange(prefix, strip=True) + if not previous: + pk = set() + else: + pk = set(previous.keys()) + ck = set(mapping.keys()) + delta = DeltaSet() + + # added + for k in ck.difference(pk): + delta[k] = Delta(None, mapping[k]) + + # removed + for k in pk.difference(ck): + delta[k] = Delta(previous[k], None) + + # changed + for k in pk.intersection(ck): + c = mapping[k] + p = previous[k] + if c != p: + delta[k] = Delta(p, c) + + return delta + + @contextlib.contextmanager + def hook_scope(self, name=""): + """Scope all future interactions to the current hook execution + revision.""" + assert not self.revision + self.cursor.execute( + 'insert into hooks (hook, date) values (?, ?)', + (name or sys.argv[0], + datetime.datetime.utcnow().isoformat())) + self.revision = self.cursor.lastrowid + try: + yield self.revision + self.revision = None + except: + self.flush(False) + self.revision = None + raise + else: + self.flush() + + def flush(self, save=True): + if save: + self.conn.commit() + elif self._closed: + return + else: + self.conn.rollback() + + def _init(self): + self.cursor.execute(''' + create table if not exists kv ( + key text, + data text, + primary key (key) + )''') + self.cursor.execute(''' + create table if not exists kv_revisions ( + key text, + revision integer, + data text, + primary key (key, revision) + )''') + self.cursor.execute(''' + create table if not exists hooks ( + version integer primary key autoincrement, + hook text, + date text + )''') + self.conn.commit() + + def gethistory(self, key, deserialize=False): + self.cursor.execute( + ''' + select kv.revision, kv.key, kv.data, h.hook, h.date + from kv_revisions kv, + hooks h + where kv.key=? + and kv.revision = h.version + ''', [key]) + if deserialize is False: + return self.cursor.fetchall() + return map(_parse_history, self.cursor.fetchall()) + + def debug(self, fh=sys.stderr): + self.cursor.execute('select * from kv') + pprint.pprint(self.cursor.fetchall(), stream=fh) + self.cursor.execute('select * from kv_revisions') + pprint.pprint(self.cursor.fetchall(), stream=fh) + + +def _parse_history(d): + return (d[0], d[1], json.loads(d[2]), d[3], + datetime.datetime.strptime(d[-1], "%Y-%m-%dT%H:%M:%S.%f")) + + +class HookData(object): + """Simple integration for existing hook exec frameworks. + + Records all unit information, and stores deltas for processing + by the hook. + + Sample:: + + from charmhelper.core import hookenv, unitdata + + changes = unitdata.HookData() + db = unitdata.kv() + hooks = hookenv.Hooks() + + @hooks.hook + def config_changed(): + # View all changes to configuration + for changed, (prev, cur) in changes.conf.items(): + print('config changed', changed, + 'previous value', prev, + 'current value', cur) + + # Get some unit specific bookeeping + if not db.get('pkg_key'): + key = urllib.urlopen('https://example.com/pkg_key').read() + db.set('pkg_key', key) + + if __name__ == '__main__': + with changes(): + hook.execute() + + """ + def __init__(self): + self.kv = kv() + self.conf = None + self.rels = None + + @contextlib.contextmanager + def __call__(self): + from charmhelpers.core import hookenv + hook_name = hookenv.hook_name() + + with self.kv.hook_scope(hook_name): + self._record_charm_version(hookenv.charm_dir()) + delta_config, delta_relation = self._record_hook(hookenv) + yield self.kv, delta_config, delta_relation + + def _record_charm_version(self, charm_dir): + # Record revisions.. charm revisions are meaningless + # to charm authors as they don't control the revision. + # so logic dependnent on revision is not particularly + # useful, however it is useful for debugging analysis. + charm_rev = open( + os.path.join(charm_dir, 'revision')).read().strip() + charm_rev = charm_rev or '0' + revs = self.kv.get('charm_revisions', []) + if charm_rev not in revs: + revs.append(charm_rev.strip() or '0') + self.kv.set('charm_revisions', revs) + + def _record_hook(self, hookenv): + data = hookenv.execution_environment() + self.conf = conf_delta = self.kv.delta(data['conf'], 'config') + self.rels = rels_delta = self.kv.delta(data['rels'], 'rels') + self.kv.set('env', dict(data['env'])) + self.kv.set('unit', data['unit']) + self.kv.set('relid', data.get('relid')) + return conf_delta, rels_delta + + +class Record(dict): + + __slots__ = () + + def __getattr__(self, k): + if k in self: + return self[k] + raise AttributeError(k) + + +class DeltaSet(Record): + + __slots__ = () + + +Delta = collections.namedtuple('Delta', ['previous', 'current']) + + +_KV = None + + +def kv(): + global _KV + if _KV is None: + _KV = Storage() + return _KV diff --git a/contrail-analytics/hooks/charmhelpers/fetch/__init__.py b/contrail-analytics/hooks/charmhelpers/fetch/__init__.py new file mode 100644 index 0000000..480a627 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/__init__.py @@ -0,0 +1,205 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from charmhelpers.osplatform import get_platform +from yaml import safe_load +from charmhelpers.core.hookenv import ( + config, + log, +) + +import six +if six.PY3: + from urllib.parse import urlparse, urlunparse +else: + from urlparse import urlparse, urlunparse + + +# The order of this list is very important. Handlers should be listed in from +# least- to most-specific URL matching. +FETCH_HANDLERS = ( + 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler', + 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler', + 'charmhelpers.fetch.giturl.GitUrlFetchHandler', +) + + +class SourceConfigError(Exception): + pass + + +class UnhandledSource(Exception): + pass + + +class AptLockError(Exception): + pass + + +class GPGKeyError(Exception): + """Exception occurs when a GPG key cannot be fetched or used. The message + indicates what the problem is. + """ + pass + + +class BaseFetchHandler(object): + + """Base class for FetchHandler implementations in fetch plugins""" + + def can_handle(self, source): + """Returns True if the source can be handled. Otherwise returns + a string explaining why it cannot""" + return "Wrong source type" + + def install(self, source): + """Try to download and unpack the source. Return the path to the + unpacked files or raise UnhandledSource.""" + raise UnhandledSource("Wrong source type {}".format(source)) + + def parse_url(self, url): + return urlparse(url) + + def base_url(self, url): + """Return url without querystring or fragment""" + parts = list(self.parse_url(url)) + parts[4:] = ['' for i in parts[4:]] + return urlunparse(parts) + + +__platform__ = get_platform() +module = "charmhelpers.fetch.%s" % __platform__ +fetch = importlib.import_module(module) + +filter_installed_packages = fetch.filter_installed_packages +install = fetch.apt_install +upgrade = fetch.apt_upgrade +update = _fetch_update = fetch.apt_update +purge = fetch.apt_purge +add_source = fetch.add_source + +if __platform__ == "ubuntu": + apt_cache = fetch.apt_cache + apt_install = fetch.apt_install + apt_update = fetch.apt_update + apt_upgrade = fetch.apt_upgrade + apt_purge = fetch.apt_purge + apt_mark = fetch.apt_mark + apt_hold = fetch.apt_hold + apt_unhold = fetch.apt_unhold + import_key = fetch.import_key + get_upstream_version = fetch.get_upstream_version +elif __platform__ == "centos": + yum_search = fetch.yum_search + + +def configure_sources(update=False, + sources_var='install_sources', + keys_var='install_keys'): + """Configure multiple sources from charm configuration. + + The lists are encoded as yaml fragments in the configuration. + The fragment needs to be included as a string. Sources and their + corresponding keys are of the types supported by add_source(). + + Example config: + install_sources: | + - "ppa:foo" + - "http://example.com/repo precise main" + install_keys: | + - null + - "a1b2c3d4" + + Note that 'null' (a.k.a. None) should not be quoted. + """ + sources = safe_load((config(sources_var) or '').strip()) or [] + keys = safe_load((config(keys_var) or '').strip()) or None + + if isinstance(sources, six.string_types): + sources = [sources] + + if keys is None: + for source in sources: + add_source(source, None) + else: + if isinstance(keys, six.string_types): + keys = [keys] + + if len(sources) != len(keys): + raise SourceConfigError( + 'Install sources and keys lists are different lengths') + for source, key in zip(sources, keys): + add_source(source, key) + if update: + _fetch_update(fatal=True) + + +def install_remote(source, *args, **kwargs): + """Install a file tree from a remote source. + + The specified source should be a url of the form: + scheme://[host]/path[#[option=value][&...]] + + Schemes supported are based on this modules submodules. + Options supported are submodule-specific. + Additional arguments are passed through to the submodule. + + For example:: + + dest = install_remote('http://example.com/archive.tgz', + checksum='deadbeef', + hash_type='sha1') + + This will download `archive.tgz`, validate it using SHA1 and, if + the file is ok, extract it and return the directory in which it + was extracted. If the checksum fails, it will raise + :class:`charmhelpers.core.host.ChecksumError`. + """ + # We ONLY check for True here because can_handle may return a string + # explaining why it can't handle a given source. + handlers = [h for h in plugins() if h.can_handle(source) is True] + for handler in handlers: + try: + return handler.install(source, *args, **kwargs) + except UnhandledSource as e: + log('Install source attempt unsuccessful: {}'.format(e), + level='WARNING') + raise UnhandledSource("No handler found for source {}".format(source)) + + +def install_from_config(config_var_name): + """Install a file from config.""" + charm_config = config() + source = charm_config[config_var_name] + return install_remote(source) + + +def plugins(fetch_handlers=None): + if not fetch_handlers: + fetch_handlers = FETCH_HANDLERS + plugin_list = [] + for handler_name in fetch_handlers: + package, classname = handler_name.rsplit('.', 1) + try: + handler_class = getattr( + importlib.import_module(package), + classname) + plugin_list.append(handler_class()) + except NotImplementedError: + # Skip missing plugins so that they can be ommitted from + # installation if desired + log("FetchHandler {} not found, skipping plugin".format( + handler_name)) + return plugin_list diff --git a/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py b/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py new file mode 100644 index 0000000..dd24f9e --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py @@ -0,0 +1,165 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import hashlib +import re + +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource +) +from charmhelpers.payload.archive import ( + get_archive_handler, + extract, +) +from charmhelpers.core.host import mkdir, check_hash + +import six +if six.PY3: + from urllib.request import ( + build_opener, install_opener, urlopen, urlretrieve, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + ) + from urllib.parse import urlparse, urlunparse, parse_qs + from urllib.error import URLError +else: + from urllib import urlretrieve + from urllib2 import ( + build_opener, install_opener, urlopen, + HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, + URLError + ) + from urlparse import urlparse, urlunparse, parse_qs + + +def splituser(host): + '''urllib.splituser(), but six's support of this seems broken''' + _userprog = re.compile('^(.*)@(.*)$') + match = _userprog.match(host) + if match: + return match.group(1, 2) + return None, host + + +def splitpasswd(user): + '''urllib.splitpasswd(), but six's support of this is missing''' + _passwdprog = re.compile('^([^:]*):(.*)$', re.S) + match = _passwdprog.match(user) + if match: + return match.group(1, 2) + return user, None + + +class ArchiveUrlFetchHandler(BaseFetchHandler): + """ + Handler to download archive files from arbitrary URLs. + + Can fetch from http, https, ftp, and file URLs. + + Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files. + + Installs the contents of the archive in $CHARM_DIR/fetched/. + """ + def can_handle(self, source): + url_parts = self.parse_url(source) + if url_parts.scheme not in ('http', 'https', 'ftp', 'file'): + # XXX: Why is this returning a boolean and a string? It's + # doomed to fail since "bool(can_handle('foo://'))" will be True. + return "Wrong source type" + if get_archive_handler(self.base_url(source)): + return True + return False + + def download(self, source, dest): + """ + Download an archive file. + + :param str source: URL pointing to an archive file. + :param str dest: Local path location to download archive file to. + """ + # propogate all exceptions + # URLError, OSError, etc + proto, netloc, path, params, query, fragment = urlparse(source) + if proto in ('http', 'https'): + auth, barehost = splituser(netloc) + if auth is not None: + source = urlunparse((proto, barehost, path, params, query, fragment)) + username, password = splitpasswd(auth) + passman = HTTPPasswordMgrWithDefaultRealm() + # Realm is set to None in add_password to force the username and password + # to be used whatever the realm + passman.add_password(None, source, username, password) + authhandler = HTTPBasicAuthHandler(passman) + opener = build_opener(authhandler) + install_opener(opener) + response = urlopen(source) + try: + with open(dest, 'wb') as dest_file: + dest_file.write(response.read()) + except Exception as e: + if os.path.isfile(dest): + os.unlink(dest) + raise e + + # Mandatory file validation via Sha1 or MD5 hashing. + def download_and_validate(self, url, hashsum, validate="sha1"): + tempfile, headers = urlretrieve(url) + check_hash(tempfile, hashsum, validate) + return tempfile + + def install(self, source, dest=None, checksum=None, hash_type='sha1'): + """ + Download and install an archive file, with optional checksum validation. + + The checksum can also be given on the `source` URL's fragment. + For example:: + + handler.install('http://example.com/file.tgz#sha1=deadbeef') + + :param str source: URL pointing to an archive file. + :param str dest: Local destination path to install to. If not given, + installs to `$CHARM_DIR/archives/archive_file_name`. + :param str checksum: If given, validate the archive file after download. + :param str hash_type: Algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + + """ + url_parts = self.parse_url(source) + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') + if not os.path.exists(dest_dir): + mkdir(dest_dir, perms=0o755) + dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path)) + try: + self.download(source, dld_file) + except URLError as e: + raise UnhandledSource(e.reason) + except OSError as e: + raise UnhandledSource(e.strerror) + options = parse_qs(url_parts.fragment) + for key, value in options.items(): + if not six.PY3: + algorithms = hashlib.algorithms + else: + algorithms = hashlib.algorithms_available + if key in algorithms: + if len(value) != 1: + raise TypeError( + "Expected 1 hash value, not %d" % len(value)) + expected = value[0] + check_hash(dld_file, expected, key) + if checksum: + check_hash(dld_file, checksum, hash_type) + return extract(dld_file, dest) diff --git a/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py b/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py new file mode 100644 index 0000000..07cd029 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py @@ -0,0 +1,76 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from subprocess import check_call +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource, + filter_installed_packages, + install, +) +from charmhelpers.core.host import mkdir + + +if filter_installed_packages(['bzr']) != []: + install(['bzr']) + if filter_installed_packages(['bzr']) != []: + raise NotImplementedError('Unable to install bzr') + + +class BzrUrlFetchHandler(BaseFetchHandler): + """Handler for bazaar branches via generic and lp URLs.""" + + def can_handle(self, source): + url_parts = self.parse_url(source) + if url_parts.scheme not in ('bzr+ssh', 'lp', ''): + return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.bzr')) + else: + return True + + def branch(self, source, dest, revno=None): + if not self.can_handle(source): + raise UnhandledSource("Cannot handle {}".format(source)) + cmd_opts = [] + if revno: + cmd_opts += ['-r', str(revno)] + if os.path.exists(dest): + cmd = ['bzr', 'pull'] + cmd += cmd_opts + cmd += ['--overwrite', '-d', dest, source] + else: + cmd = ['bzr', 'branch'] + cmd += cmd_opts + cmd += [source, dest] + check_call(cmd) + + def install(self, source, dest=None, revno=None): + url_parts = self.parse_url(source) + branch_name = url_parts.path.strip("/").split("/")[-1] + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + + if dest and not os.path.exists(dest): + mkdir(dest, perms=0o755) + + try: + self.branch(source, dest_dir, revno) + except OSError as e: + raise UnhandledSource(e.strerror) + return dest_dir diff --git a/contrail-analytics/hooks/charmhelpers/fetch/centos.py b/contrail-analytics/hooks/charmhelpers/fetch/centos.py new file mode 100644 index 0000000..a91dcff --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/centos.py @@ -0,0 +1,171 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import os +import time +import six +import yum + +from tempfile import NamedTemporaryFile +from charmhelpers.core.hookenv import log + +YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM. +YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks. +YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +def filter_installed_packages(packages): + """Return a list of packages that require installation.""" + yb = yum.YumBase() + package_list = yb.doPackageLists() + temp_cache = {p.base_package_name: 1 for p in package_list['installed']} + + _pkgs = [p for p in packages if not temp_cache.get(p, False)] + return _pkgs + + +def install(packages, options=None, fatal=False): + """Install one or more packages.""" + cmd = ['yum', '--assumeyes'] + if options is not None: + cmd.extend(options) + cmd.append('install') + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Installing {} with options: {}".format(packages, + options)) + _run_yum_command(cmd, fatal) + + +def upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages.""" + cmd = ['yum', '--assumeyes'] + if options is not None: + cmd.extend(options) + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + _run_yum_command(cmd, fatal) + + +def update(fatal=False): + """Update local yum cache.""" + cmd = ['yum', '--assumeyes', 'update'] + log("Update with fatal: {}".format(fatal)) + _run_yum_command(cmd, fatal) + + +def purge(packages, fatal=False): + """Purge one or more packages.""" + cmd = ['yum', '--assumeyes', 'remove'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Purging {}".format(packages)) + _run_yum_command(cmd, fatal) + + +def yum_search(packages): + """Search for a package.""" + output = {} + cmd = ['yum', 'search'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Searching for {}".format(packages)) + result = subprocess.check_output(cmd) + for package in list(packages): + output[package] = package in result + return output + + +def add_source(source, key=None): + """Add a package source to this system. + + @param source: a URL with a rpm package + + @param key: A key to be added to the system's keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. + """ + if source is None: + log('Source is not present. Skipping') + return + + if source.startswith('http'): + directory = '/etc/yum.repos.d/' + for filename in os.listdir(directory): + with open(directory + filename, 'r') as rpm_file: + if source in rpm_file.read(): + break + else: + log("Add source: {!r}".format(source)) + # write in the charms.repo + with open(directory + 'Charms.repo', 'a') as rpm_file: + rpm_file.write('[%s]\n' % source[7:].replace('/', '_')) + rpm_file.write('name=%s\n' % source[7:]) + rpm_file.write('baseurl=%s\n\n' % source) + else: + log("Unknown source: {!r}".format(source)) + + if key: + if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key: + with NamedTemporaryFile('w+') as key_file: + key_file.write(key) + key_file.flush() + key_file.seek(0) + subprocess.check_call(['rpm', '--import', key_file.name]) + else: + subprocess.check_call(['rpm', '--import', key]) + + +def _run_yum_command(cmd, fatal=False): + """Run an YUM command. + + Checks the output and retry if the fatal flag is set to True. + + :param: cmd: str: The yum command to run. + :param: fatal: bool: Whether the command's output should be checked and + retried. + """ + env = os.environ.copy() + + if fatal: + retry_count = 0 + result = None + + # If the command is considered "fatal", we need to retry if the yum + # lock was not acquired. + + while result is None or result == YUM_NO_LOCK: + try: + result = subprocess.check_call(cmd, env=env) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > YUM_NO_LOCK_RETRY_COUNT: + raise + result = e.returncode + log("Couldn't acquire YUM lock. Will retry in {} seconds." + "".format(YUM_NO_LOCK_RETRY_DELAY)) + time.sleep(YUM_NO_LOCK_RETRY_DELAY) + + else: + subprocess.call(cmd, env=env) diff --git a/contrail-analytics/hooks/charmhelpers/fetch/giturl.py b/contrail-analytics/hooks/charmhelpers/fetch/giturl.py new file mode 100644 index 0000000..4cf21bc --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/giturl.py @@ -0,0 +1,69 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from subprocess import check_call, CalledProcessError +from charmhelpers.fetch import ( + BaseFetchHandler, + UnhandledSource, + filter_installed_packages, + install, +) + +if filter_installed_packages(['git']) != []: + install(['git']) + if filter_installed_packages(['git']) != []: + raise NotImplementedError('Unable to install git') + + +class GitUrlFetchHandler(BaseFetchHandler): + """Handler for git branches via generic and github URLs.""" + + def can_handle(self, source): + url_parts = self.parse_url(source) + # TODO (mattyw) no support for ssh git@ yet + if url_parts.scheme not in ('http', 'https', 'git', ''): + return False + elif not url_parts.scheme: + return os.path.exists(os.path.join(source, '.git')) + else: + return True + + def clone(self, source, dest, branch="master", depth=None): + if not self.can_handle(source): + raise UnhandledSource("Cannot handle {}".format(source)) + + if os.path.exists(dest): + cmd = ['git', '-C', dest, 'pull', source, branch] + else: + cmd = ['git', 'clone', source, dest, '--branch', branch] + if depth: + cmd.extend(['--depth', depth]) + check_call(cmd) + + def install(self, source, branch="master", dest=None, depth=None): + url_parts = self.parse_url(source) + branch_name = url_parts.path.strip("/").split("/")[-1] + if dest: + dest_dir = os.path.join(dest, branch_name) + else: + dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", + branch_name) + try: + self.clone(source, dest_dir, branch, depth) + except CalledProcessError as e: + raise UnhandledSource(e) + except OSError as e: + raise UnhandledSource(e.strerror) + return dest_dir diff --git a/contrail-analytics/hooks/charmhelpers/fetch/snap.py b/contrail-analytics/hooks/charmhelpers/fetch/snap.py new file mode 100644 index 0000000..23c707b --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/snap.py @@ -0,0 +1,122 @@ +# Copyright 2014-2017 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Charm helpers snap for classic charms. + +If writing reactive charms, use the snap layer: +https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html +""" +import subprocess +from os import environ +from time import sleep +from charmhelpers.core.hookenv import log + +__author__ = 'Joseph Borg <joseph.borg@canonical.com>' + +SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved). +SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks. +SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times. + + +class CouldNotAcquireLockException(Exception): + pass + + +def _snap_exec(commands): + """ + Execute snap commands. + + :param commands: List commands + :return: Integer exit code + """ + assert type(commands) == list + + retry_count = 0 + return_code = None + + while return_code is None or return_code == SNAP_NO_LOCK: + try: + return_code = subprocess.check_call(['snap'] + commands, env=environ) + except subprocess.CalledProcessError as e: + retry_count += + 1 + if retry_count > SNAP_NO_LOCK_RETRY_COUNT: + raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT) + return_code = e.returncode + log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN') + sleep(SNAP_NO_LOCK_RETRY_DELAY) + + return return_code + + +def snap_install(packages, *flags): + """ + Install a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to install command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Installing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with option(s) "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['install'] + flags + packages) + + +def snap_remove(packages, *flags): + """ + Remove a snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to remove command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Removing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['remove'] + flags + packages) + + +def snap_refresh(packages, *flags): + """ + Refresh / Update snap package. + + :param packages: String or List String package name + :param flags: List String flags to pass to refresh command + :return: Integer return code from snap + """ + if type(packages) is not list: + packages = [packages] + + flags = list(flags) + + message = 'Refreshing snap(s) "%s"' % ', '.join(packages) + if flags: + message += ' with options "%s"' % ', '.join(flags) + + log(message, level='INFO') + return _snap_exec(['refresh'] + flags + packages) diff --git a/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py b/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py new file mode 100644 index 0000000..57b5fb6 --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py @@ -0,0 +1,568 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict +import os +import platform +import re +import six +import time +import subprocess +from tempfile import NamedTemporaryFile + +from charmhelpers.core.host import ( + lsb_release +) +from charmhelpers.core.hookenv import ( + log, + DEBUG, +) +from charmhelpers.fetch import SourceConfigError, GPGKeyError + +PROPOSED_POCKET = ( + "# Proposed\n" + "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe " + "multiverse restricted\n") +PROPOSED_PORTS_POCKET = ( + "# Proposed\n" + "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe " + "multiverse restricted\n") +# Only supports 64bit and ppc64 at the moment. +ARCH_TO_PROPOSED_POCKET = { + 'x86_64': PROPOSED_POCKET, + 'ppc64le': PROPOSED_PORTS_POCKET, + 'aarch64': PROPOSED_PORTS_POCKET, +} +CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu" +CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA' +CLOUD_ARCHIVE = """# Ubuntu Cloud Archive +deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main +""" +CLOUD_ARCHIVE_POCKETS = { + # Folsom + 'folsom': 'precise-updates/folsom', + 'folsom/updates': 'precise-updates/folsom', + 'precise-folsom': 'precise-updates/folsom', + 'precise-folsom/updates': 'precise-updates/folsom', + 'precise-updates/folsom': 'precise-updates/folsom', + 'folsom/proposed': 'precise-proposed/folsom', + 'precise-folsom/proposed': 'precise-proposed/folsom', + 'precise-proposed/folsom': 'precise-proposed/folsom', + # Grizzly + 'grizzly': 'precise-updates/grizzly', + 'grizzly/updates': 'precise-updates/grizzly', + 'precise-grizzly': 'precise-updates/grizzly', + 'precise-grizzly/updates': 'precise-updates/grizzly', + 'precise-updates/grizzly': 'precise-updates/grizzly', + 'grizzly/proposed': 'precise-proposed/grizzly', + 'precise-grizzly/proposed': 'precise-proposed/grizzly', + 'precise-proposed/grizzly': 'precise-proposed/grizzly', + # Havana + 'havana': 'precise-updates/havana', + 'havana/updates': 'precise-updates/havana', + 'precise-havana': 'precise-updates/havana', + 'precise-havana/updates': 'precise-updates/havana', + 'precise-updates/havana': 'precise-updates/havana', + 'havana/proposed': 'precise-proposed/havana', + 'precise-havana/proposed': 'precise-proposed/havana', + 'precise-proposed/havana': 'precise-proposed/havana', + # Icehouse + 'icehouse': 'precise-updates/icehouse', + 'icehouse/updates': 'precise-updates/icehouse', + 'precise-icehouse': 'precise-updates/icehouse', + 'precise-icehouse/updates': 'precise-updates/icehouse', + 'precise-updates/icehouse': 'precise-updates/icehouse', + 'icehouse/proposed': 'precise-proposed/icehouse', + 'precise-icehouse/proposed': 'precise-proposed/icehouse', + 'precise-proposed/icehouse': 'precise-proposed/icehouse', + # Juno + 'juno': 'trusty-updates/juno', + 'juno/updates': 'trusty-updates/juno', + 'trusty-juno': 'trusty-updates/juno', + 'trusty-juno/updates': 'trusty-updates/juno', + 'trusty-updates/juno': 'trusty-updates/juno', + 'juno/proposed': 'trusty-proposed/juno', + 'trusty-juno/proposed': 'trusty-proposed/juno', + 'trusty-proposed/juno': 'trusty-proposed/juno', + # Kilo + 'kilo': 'trusty-updates/kilo', + 'kilo/updates': 'trusty-updates/kilo', + 'trusty-kilo': 'trusty-updates/kilo', + 'trusty-kilo/updates': 'trusty-updates/kilo', + 'trusty-updates/kilo': 'trusty-updates/kilo', + 'kilo/proposed': 'trusty-proposed/kilo', + 'trusty-kilo/proposed': 'trusty-proposed/kilo', + 'trusty-proposed/kilo': 'trusty-proposed/kilo', + # Liberty + 'liberty': 'trusty-updates/liberty', + 'liberty/updates': 'trusty-updates/liberty', + 'trusty-liberty': 'trusty-updates/liberty', + 'trusty-liberty/updates': 'trusty-updates/liberty', + 'trusty-updates/liberty': 'trusty-updates/liberty', + 'liberty/proposed': 'trusty-proposed/liberty', + 'trusty-liberty/proposed': 'trusty-proposed/liberty', + 'trusty-proposed/liberty': 'trusty-proposed/liberty', + # Mitaka + 'mitaka': 'trusty-updates/mitaka', + 'mitaka/updates': 'trusty-updates/mitaka', + 'trusty-mitaka': 'trusty-updates/mitaka', + 'trusty-mitaka/updates': 'trusty-updates/mitaka', + 'trusty-updates/mitaka': 'trusty-updates/mitaka', + 'mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-mitaka/proposed': 'trusty-proposed/mitaka', + 'trusty-proposed/mitaka': 'trusty-proposed/mitaka', + # Newton + 'newton': 'xenial-updates/newton', + 'newton/updates': 'xenial-updates/newton', + 'xenial-newton': 'xenial-updates/newton', + 'xenial-newton/updates': 'xenial-updates/newton', + 'xenial-updates/newton': 'xenial-updates/newton', + 'newton/proposed': 'xenial-proposed/newton', + 'xenial-newton/proposed': 'xenial-proposed/newton', + 'xenial-proposed/newton': 'xenial-proposed/newton', + # Ocata + 'ocata': 'xenial-updates/ocata', + 'ocata/updates': 'xenial-updates/ocata', + 'xenial-ocata': 'xenial-updates/ocata', + 'xenial-ocata/updates': 'xenial-updates/ocata', + 'xenial-updates/ocata': 'xenial-updates/ocata', + 'ocata/proposed': 'xenial-proposed/ocata', + 'xenial-ocata/proposed': 'xenial-proposed/ocata', + 'xenial-ocata/newton': 'xenial-proposed/ocata', + # Pike + 'pike': 'xenial-updates/pike', + 'xenial-pike': 'xenial-updates/pike', + 'xenial-pike/updates': 'xenial-updates/pike', + 'xenial-updates/pike': 'xenial-updates/pike', + 'pike/proposed': 'xenial-proposed/pike', + 'xenial-pike/proposed': 'xenial-proposed/pike', + 'xenial-pike/newton': 'xenial-proposed/pike', + # Queens + 'queens': 'xenial-updates/queens', + 'xenial-queens': 'xenial-updates/queens', + 'xenial-queens/updates': 'xenial-updates/queens', + 'xenial-updates/queens': 'xenial-updates/queens', + 'queens/proposed': 'xenial-proposed/queens', + 'xenial-queens/proposed': 'xenial-proposed/queens', + 'xenial-queens/newton': 'xenial-proposed/queens', +} + + +APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT. +CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries. +CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times. + + +def filter_installed_packages(packages): + """Return a list of packages that require installation.""" + cache = apt_cache() + _pkgs = [] + for package in packages: + try: + p = cache[package] + p.current_ver or _pkgs.append(package) + except KeyError: + log('Package {} has no installation candidate.'.format(package), + level='WARNING') + _pkgs.append(package) + return _pkgs + + +def apt_cache(in_memory=True, progress=None): + """Build and return an apt cache.""" + from apt import apt_pkg + apt_pkg.init() + if in_memory: + apt_pkg.config.set("Dir::Cache::pkgcache", "") + apt_pkg.config.set("Dir::Cache::srcpkgcache", "") + return apt_pkg.Cache(progress) + + +def apt_install(packages, options=None, fatal=False): + """Install one or more packages.""" + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + cmd.append('install') + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Installing {} with options: {}".format(packages, + options)) + _run_apt_command(cmd, fatal) + + +def apt_upgrade(options=None, fatal=False, dist=False): + """Upgrade all packages.""" + if options is None: + options = ['--option=Dpkg::Options::=--force-confold'] + + cmd = ['apt-get', '--assume-yes'] + cmd.extend(options) + if dist: + cmd.append('dist-upgrade') + else: + cmd.append('upgrade') + log("Upgrading with options: {}".format(options)) + _run_apt_command(cmd, fatal) + + +def apt_update(fatal=False): + """Update local apt cache.""" + cmd = ['apt-get', 'update'] + _run_apt_command(cmd, fatal) + + +def apt_purge(packages, fatal=False): + """Purge one or more packages.""" + cmd = ['apt-get', '--assume-yes', 'purge'] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + log("Purging {}".format(packages)) + _run_apt_command(cmd, fatal) + + +def apt_mark(packages, mark, fatal=False): + """Flag one or more packages using apt-mark.""" + log("Marking {} as {}".format(packages, mark)) + cmd = ['apt-mark', mark] + if isinstance(packages, six.string_types): + cmd.append(packages) + else: + cmd.extend(packages) + + if fatal: + subprocess.check_call(cmd, universal_newlines=True) + else: + subprocess.call(cmd, universal_newlines=True) + + +def apt_hold(packages, fatal=False): + return apt_mark(packages, 'hold', fatal=fatal) + + +def apt_unhold(packages, fatal=False): + return apt_mark(packages, 'unhold', fatal=fatal) + + +def import_key(keyid): + """Import a key in either ASCII Armor or Radix64 format. + + `keyid` is either the keyid to fetch from a PGP server, or + the key in ASCII armor foramt. + + :param keyid: String of key (or key id). + :raises: GPGKeyError if the key could not be imported + """ + key = keyid.strip() + if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and + key.endswith('-----END PGP PUBLIC KEY BLOCK-----')): + log("PGP key found (looks like ASCII Armor format)", level=DEBUG) + log("Importing ASCII Armor PGP key", level=DEBUG) + with NamedTemporaryFile() as keyfile: + with open(keyfile.name, 'w') as fd: + fd.write(key) + fd.write("\n") + cmd = ['apt-key', 'add', keyfile.name] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + else: + log("PGP key found (looks like Radix64 format)", level=DEBUG) + log("Importing PGP key from keyserver", level=DEBUG) + cmd = ['apt-key', 'adv', '--keyserver', + 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key] + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + error = "Error importing PGP key '{}'".format(key) + log(error) + raise GPGKeyError(error) + + +def add_source(source, key=None, fail_invalid=False): + """Add a package source to this system. + + @param source: a URL or sources.list entry, as supported by + add-apt-repository(1). Examples:: + + ppa:charmers/example + deb https://stub:key@private.example.com/ubuntu trusty main + + In addition: + 'proposed:' may be used to enable the standard 'proposed' + pocket for the release. + 'cloud:' may be used to activate official cloud archive pockets, + such as 'cloud:icehouse' + 'distro' may be used as a noop + + Full list of source specifications supported by the function are: + + 'distro': A NOP; i.e. it has no effect. + 'proposed': the proposed deb spec [2] is wrtten to + /etc/apt/sources.list/proposed + 'distro-proposed': adds <version>-proposed to the debs [2] + 'ppa:<ppa-name>': add-apt-repository --yes <ppa_name> + 'deb <deb-spec>': add-apt-repository --yes deb <deb-spec> + 'http://....': add-apt-repository --yes http://... + 'cloud-archive:<spec>': add-apt-repository -yes cloud-archive:<spec> + 'cloud:<release>[-staging]': specify a Cloud Archive pocket <release> with + optional staging version. If staging is used then the staging PPA [2] + with be used. If staging is NOT used then the cloud archive [3] will be + added, and the 'ubuntu-cloud-keyring' package will be added for the + current distro. + + Otherwise the source is not recognised and this is logged to the juju log. + However, no error is raised, unless sys_error_on_exit is True. + + [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main + where {} is replaced with the derived pocket name. + [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \ + main universe multiverse restricted + where {} is replaced with the lsb_release codename (e.g. xenial) + [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu <pocket> + to /etc/apt/sources.list.d/cloud-archive-list + + @param key: A key to be added to the system's APT keyring and used + to verify the signatures on packages. Ideally, this should be an + ASCII format GPG public key including the block headers. A GPG key + id may also be used, but be aware that only insecure protocols are + available to retrieve the actual public key from a public keyserver + placing your Juju environment at risk. ppa and cloud archive keys + are securely added automtically, so sould not be provided. + + @param fail_invalid: (boolean) if True, then the function raises a + SourceConfigError is there is no matching installation source. + + @raises SourceConfigError() if for cloud:<pocket>, the <pocket> is not a + valid pocket in CLOUD_ARCHIVE_POCKETS + """ + _mapping = OrderedDict([ + (r"^distro$", lambda: None), # This is a NOP + (r"^(?:proposed|distro-proposed)$", _add_proposed), + (r"^cloud-archive:(.*)$", _add_apt_repository), + (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository), + (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging), + (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check), + (r"^cloud:(.*)$", _add_cloud_pocket), + ]) + if source is None: + source = '' + for r, fn in six.iteritems(_mapping): + m = re.match(r, source) + if m: + # call the assoicated function with the captured groups + # raises SourceConfigError on error. + fn(*m.groups()) + if key: + try: + import_key(key) + except GPGKeyError as e: + raise SourceConfigError(str(e)) + break + else: + # nothing matched. log an error and maybe sys.exit + err = "Unknown source: {!r}".format(source) + log(err) + if fail_invalid: + raise SourceConfigError(err) + + +def _add_proposed(): + """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list + + Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for + the deb line. + + For intel architecutres PROPOSED_POCKET is used for the release, but for + other architectures PROPOSED_PORTS_POCKET is used for the release. + """ + release = lsb_release()['DISTRIB_CODENAME'] + arch = platform.machine() + if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET): + raise SourceConfigError("Arch {} not supported for (distro-)proposed" + .format(arch)) + with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt: + apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release)) + + +def _add_apt_repository(spec): + """Add the spec using add_apt_repository + + :param spec: the parameter to pass to add_apt_repository + """ + _run_with_retries(['add-apt-repository', '--yes', spec]) + + +def _add_cloud_pocket(pocket): + """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list + + Note that this overwrites the existing file if there is one. + + This function also converts the simple pocket in to the actual pocket using + the CLOUD_ARCHIVE_POCKETS mapping. + + :param pocket: string representing the pocket to add a deb spec for. + :raises: SourceConfigError if the cloud pocket doesn't exist or the + requested release doesn't match the current distro version. + """ + apt_install(filter_installed_packages(['ubuntu-cloud-keyring']), + fatal=True) + if pocket not in CLOUD_ARCHIVE_POCKETS: + raise SourceConfigError( + 'Unsupported cloud: source option %s' % + pocket) + actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket] + with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt: + apt.write(CLOUD_ARCHIVE.format(actual_pocket)) + + +def _add_cloud_staging(cloud_archive_release, openstack_release): + """Add the cloud staging repository which is in + ppa:ubuntu-cloud-archive/<openstack_release>-staging + + This function checks that the cloud_archive_release matches the current + codename for the distro that charm is being installed on. + + :param cloud_archive_release: string, codename for the release. + :param openstack_release: String, codename for the openstack release. + :raises: SourceConfigError if the cloud_archive_release doesn't match the + current version of the os. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release) + cmd = 'add-apt-repository -y {}'.format(ppa) + _run_with_retries(cmd.split(' ')) + + +def _add_cloud_distro_check(cloud_archive_release, openstack_release): + """Add the cloud pocket, but also check the cloud_archive_release against + the current distro, and use the openstack_release as the full lookup. + + This just calls _add_cloud_pocket() with the openstack_release as pocket + to get the correct cloud-archive.list for dpkg to work with. + + :param cloud_archive_release:String, codename for the distro release. + :param openstack_release: String, spec for the release to look up in the + CLOUD_ARCHIVE_POCKETS + :raises: SourceConfigError if this is the wrong distro, or the pocket spec + doesn't exist. + """ + _verify_is_ubuntu_rel(cloud_archive_release, openstack_release) + _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release)) + + +def _verify_is_ubuntu_rel(release, os_release): + """Verify that the release is in the same as the current ubuntu release. + + :param release: String, lowercase for the release. + :param os_release: String, the os_release being asked for + :raises: SourceConfigError if the release is not the same as the ubuntu + release. + """ + ubuntu_rel = lsb_release()['DISTRIB_CODENAME'] + if release != ubuntu_rel: + raise SourceConfigError( + 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu' + 'version ({})'.format(release, os_release, ubuntu_rel)) + + +def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,), + retry_message="", cmd_env=None): + """Run a command and retry until success or max_retries is reached. + + :param: cmd: str: The apt command to run. + :param: max_retries: int: The number of retries to attempt on a fatal + command. Defaults to CMD_RETRY_COUNT. + :param: retry_exitcodes: tuple: Optional additional exit codes to retry. + Defaults to retry on exit code 1. + :param: retry_message: str: Optional log prefix emitted during retries. + :param: cmd_env: dict: Environment variables to add to the command run. + """ + + env = None + kwargs = {} + if cmd_env: + env = os.environ.copy() + env.update(cmd_env) + kwargs['env'] = env + + if not retry_message: + retry_message = "Failed executing '{}'".format(" ".join(cmd)) + retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY) + + retry_count = 0 + result = None + + retry_results = (None,) + retry_exitcodes + while result in retry_results: + try: + # result = subprocess.check_call(cmd, env=env) + result = subprocess.check_call(cmd, **kwargs) + except subprocess.CalledProcessError as e: + retry_count = retry_count + 1 + if retry_count > max_retries: + raise + result = e.returncode + log(retry_message) + time.sleep(CMD_RETRY_DELAY) + + +def _run_apt_command(cmd, fatal=False): + """Run an apt command with optional retries. + + :param: cmd: str: The apt command to run. + :param: fatal: bool: Whether the command's output should be checked and + retried. + """ + # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment. + cmd_env = { + 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')} + + if fatal: + _run_with_retries( + cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,), + retry_message="Couldn't acquire DPKG lock") + else: + env = os.environ.copy() + env.update(cmd_env) + subprocess.call(cmd, env=env) + + +def get_upstream_version(package): + """Determine upstream version based on installed package + + @returns None (if not installed) or the upstream version + """ + import apt_pkg + cache = apt_cache() + try: + pkg = cache[package] + except: + # the package is unknown to the current apt cache. + return None + + if not pkg.current_ver: + # package is known, but no version is currently installed. + return None + + return apt_pkg.upstream_version(pkg.current_ver.ver_str) diff --git a/contrail-analytics/hooks/charmhelpers/osplatform.py b/contrail-analytics/hooks/charmhelpers/osplatform.py new file mode 100644 index 0000000..d9a4d5c --- /dev/null +++ b/contrail-analytics/hooks/charmhelpers/osplatform.py @@ -0,0 +1,25 @@ +import platform + + +def get_platform(): + """Return the current OS platform. + + For example: if current os platform is Ubuntu then a string "ubuntu" + will be returned (which is the name of the module). + This string is used to decide which platform module should be imported. + """ + # linux_distribution is deprecated and will be removed in Python 3.7 + # Warings *not* disabled, as we certainly need to fix this. + tuple_platform = platform.linux_distribution() + current_platform = tuple_platform[0] + if "Ubuntu" in current_platform: + return "ubuntu" + elif "CentOS" in current_platform: + return "centos" + elif "debian" in current_platform: + # Stock Python does not detect Ubuntu and instead returns debian. + # Or at least it does in some build environments like Travis CI + return "ubuntu" + else: + raise RuntimeError("This module is not supported on {}." + .format(current_platform)) diff --git a/contrail-analytics/hooks/common_utils.py b/contrail-analytics/hooks/common_utils.py new file mode 100644 index 0000000..9208ba7 --- /dev/null +++ b/contrail-analytics/hooks/common_utils.py @@ -0,0 +1,158 @@ +from base64 import b64decode +import os +from socket import gethostbyname, gethostname, gaierror +from subprocess import ( + CalledProcessError, + check_call, + check_output +) +import netifaces +import time +import platform +import json + +from charmhelpers.contrib.network.ip import get_address_in_network +from charmhelpers.core.hookenv import ( + config, + status_set, + log, + ERROR, + application_version_set, +) +from charmhelpers.core.host import write_file + +from docker_utils import ( + is_container_launched, + is_container_present, + apply_config_in_container, + get_docker_image_id, + load_docker_image, + launch_docker_image, + dpkg_version, + docker_exec, +) + +config = config() + + +def get_ip(): + network = config.get("control-network") + ip = get_address_in_network(network) if network else None + if not ip: + ip = _get_default_ip() + return ip + + +def _get_default_ip(): + if hasattr(netifaces, "gateways"): + iface = netifaces.gateways()["default"][netifaces.AF_INET][1] + else: + data = check_output("ip route | grep ^default", shell=True).split() + iface = data[data.index("dev") + 1] + return netifaces.ifaddresses(iface)[netifaces.AF_INET][0]["addr"] + + +def fix_hostname(): + hostname = gethostname() + try: + gethostbyname(hostname) + except gaierror: + ip = get_ip() + check_call(["sed", "-E", "-i", "-e", + ("/127.0.0.1[[:blank:]]+/a \\\n" + ip + " " + hostname), + "/etc/hosts"]) + + +def decode_cert(key): + val = config.get(key) + if not val: + return None + try: + return b64decode(val) + except Exception as e: + log("Couldn't decode certificate from config['{}']: {}".format( + key, str(e)), level=ERROR) + return None + + +def save_file(path, data): + if data: + fdir = os.path.dirname(path) + if not os.path.exists(fdir): + os.makedirs(fdir) + write_file(path, data, perms=0o400) + elif os.path.exists(path): + os.remove(path) + + +def update_services_status(name, services): + try: + output = docker_exec(name, "contrail-status") + except CalledProcessError as e: + log("Container is not ready to get contrail-status: " + str(e)) + status_set("waiting", "Waiting services to run in container") + return + + statuses = dict() + for line in output.splitlines()[1:]: + if len(line) == 0 or line.startswith("=="): + continue + lst = line.split() + if len(lst) < 2: + continue + srv = lst[0].split(":")[0] + statuses[srv] = (lst[1], " ".join(lst[2:])) + for srv in services: + if srv not in statuses: + status_set("waiting", srv + " is absent in the contrail-status") + return + status, desc = statuses.get(srv) + if status != "active": + workload = "waiting" if status == "initializing" else "blocked" + status_set(workload, "{} is not ready. Reason: {}" + .format(srv, desc)) + return + + status_set("active", "Unit is ready") + + +def check_run_prerequisites(name, config_name, update_config_func, services): + if is_container_launched(name): + check = True + if update_config_func: + update_config_func() + check = apply_config_in_container(name, config_name) + if check: + update_services_status(name, services) + return False + + if is_container_present(name): + status_set( + "blocked", + "Container is present but is not running. Run or remove it.") + return False + + image_id = get_docker_image_id(name) + if not image_id: + image_id = load_docker_image(name) + if not image_id: + status_set("waiting", "Awaiting for container resource") + return False + + return True + + +def run_container(name, pkg_to_check): + args = [] + if platform.linux_distribution()[2].strip() == "trusty": + args.append("--pid=host") + launch_docker_image(name, args) + + time.sleep(5) + version = dpkg_version(name, pkg_to_check) + application_version_set(version) + status_set("waiting", "Waiting services to run in container") + + +def json_loads(data, default=None): + return json.loads(data) if data else default diff --git a/contrail-analytics/hooks/config-changed b/contrail-analytics/hooks/config-changed new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/config-changed @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analytics-relation-changed b/contrail-analytics/hooks/contrail-analytics-relation-changed new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analytics-relation-changed @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analytics-relation-departed b/contrail-analytics/hooks/contrail-analytics-relation-departed new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analytics-relation-departed @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analytics-relation-joined b/contrail-analytics/hooks/contrail-analytics-relation-joined new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analytics-relation-joined @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analyticsdb-relation-changed b/contrail-analytics/hooks/contrail-analyticsdb-relation-changed new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analyticsdb-relation-changed @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analyticsdb-relation-departed b/contrail-analytics/hooks/contrail-analyticsdb-relation-departed new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analyticsdb-relation-departed @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail-analyticsdb-relation-joined b/contrail-analytics/hooks/contrail-analyticsdb-relation-joined new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/contrail-analyticsdb-relation-joined @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/contrail_analytics_hooks.py b/contrail-analytics/hooks/contrail_analytics_hooks.py new file mode 100755 index 0000000..5929290 --- /dev/null +++ b/contrail-analytics/hooks/contrail_analytics_hooks.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +import sys +import yaml + +from charmhelpers.core.hookenv import ( + Hooks, + UnregisteredHookError, + config, + log, + relation_get, + relation_ids, + related_units, + status_set, + relation_set, + local_unit, +) + +from charmhelpers.fetch import ( + apt_install, + apt_upgrade, + apt_update +) + +from contrail_analytics_utils import ( + update_charm_status, + CONTAINER_NAME, +) +from common_utils import ( + get_ip, + fix_hostname, +) +from docker_utils import ( + add_docker_repo, + DOCKER_PACKAGES, + is_container_launched, +) + + +PACKAGES = [] + +hooks = Hooks() +config = config() + + +@hooks.hook("install.real") +def install(): + status_set("maintenance", "Installing...") + + # TODO: try to remove this call + fix_hostname() + + apt_upgrade(fatal=True, dist=True) + add_docker_repo() + apt_update(fatal=False) + apt_install(PACKAGES + DOCKER_PACKAGES, fatal=True) + + update_charm_status() + + +@hooks.hook("config-changed") +def config_changed(): + if config.changed("control-network"): + settings = {'private-address': get_ip()} + rnames = ("contrail-analytics", "contrail-analyticsdb", + "analytics-cluster", "http-services") + for rname in rnames: + for rid in relation_ids(rname): + relation_set(relation_id=rid, relation_settings=settings) + + update_charm_status() + + +def _value_changed(rel_data, rel_key, cfg_key): + if rel_key not in rel_data: + # data is absent in relation. it means that remote charm doesn't + # send it due to lack of information + return False + value = rel_data[rel_key] + if value is not None and value != config.get(cfg_key): + config[cfg_key] = value + return True + elif value is None and config.get(cfg_key) is not None: + config.pop(cfg_key, None) + return True + return False + + +@hooks.hook("contrail-analytics-relation-joined") +def contrail_analytics_joined(): + settings = {"private-address": get_ip()} + relation_set(relation_settings=settings) + + +@hooks.hook("contrail-analytics-relation-changed") +def contrail_analytics_changed(): + data = relation_get() + changed = False + changed |= _value_changed(data, "auth-mode", "auth_mode") + changed |= _value_changed(data, "auth-info", "auth_info") + changed |= _value_changed(data, "orchestrator-info", "orchestrator_info") + changed |= _value_changed(data, "ssl-ca", "ssl_ca") + changed |= _value_changed(data, "ssl-cert", "ssl_cert") + changed |= _value_changed(data, "ssl-key", "ssl_key") + changed |= _value_changed(data, "rabbitmq_user", "rabbitmq_user") + changed |= _value_changed(data, "rabbitmq_password", "rabbitmq_password") + changed |= _value_changed(data, "rabbitmq_vhost", "rabbitmq_vhost") + # TODO: handle changing of all values + # TODO: set error if orchestrator is changing and container was started + if changed: + update_charm_status() + + +@hooks.hook("contrail-analytics-relation-departed") +def contrail_analytics_departed(): + units = [unit for rid in relation_ids("contrail-controller") + for unit in related_units(rid)] + if not units: + for key in ["auth_info", "auth_mode", "orchestrator_info", + "ssl_ca", "ssl_cert", "ssl_key", "rabbitmq_vhost", + "rabbitmq_user", "rabbitmq_password"]: + config.pop(key, None) + if is_container_launched(CONTAINER_NAME): + status_set( + "blocked", + "Container is present but cloud orchestrator was disappeared." + " Please kill container by yourself or restore cloud orchestrator.") + update_charm_status() + + +@hooks.hook("contrail-analyticsdb-relation-joined") +def contrail_analyticsdb_joined(): + settings = {"private-address": get_ip()} + relation_set(relation_settings=settings) + + +@hooks.hook("contrail-analyticsdb-relation-changed") +def contrail_analyticsdb_changed(): + data = relation_get() + _value_changed(data, "db-user", "db_user") + _value_changed(data, "db-password", "db_password") + update_charm_status() + + +@hooks.hook("contrail-analyticsdb-relation-departed") +def contrail_analyticsdb_departed(): + units = [unit for rid in relation_ids("contrail-analyticsdb") + for unit in related_units(rid)] + if not units: + config.pop("db_user", None) + config.pop("db_password", None) + update_charm_status() + + +@hooks.hook("analytics-cluster-relation-joined") +def analytics_cluster_joined(): + settings = {"private-address": get_ip()} + relation_set(relation_settings=settings) + + update_charm_status() + + +@hooks.hook("update-status") +def update_status(): + update_charm_status(update_config=False) + + +@hooks.hook("upgrade-charm") +def upgrade_charm(): + # NOTE: image can not be deleted if container is running. + # TODO: so think about killing the container + + # NOTE: this hook can be fired when either resource changed or charm code + # changed. so if code was changed then we may need to update config + update_charm_status() + + +def _http_services(): + name = local_unit().replace("/", "-") + addr = get_ip() + return [{"service_name": "contrail-analytics-api", + "service_host": "*", + "service_port": 8081, + "service_options": ["option nolinger", "balance roundrobin"], + "servers": [[name, addr, 8081, "check inter 2000 rise 2 fall 3"]] + }] + + +@hooks.hook("http-services-relation-joined") +def http_services_joined(): + relation_set(services=yaml.dump(_http_services())) + + +def main(): + try: + hooks.execute(sys.argv) + except UnregisteredHookError as e: + log("Unknown hook {} - skipping.".format(e)) + + +if __name__ == "__main__": + main() diff --git a/contrail-analytics/hooks/contrail_analytics_utils.py b/contrail-analytics/hooks/contrail_analytics_utils.py new file mode 100644 index 0000000..43aefec --- /dev/null +++ b/contrail-analytics/hooks/contrail_analytics_utils.py @@ -0,0 +1,163 @@ +from socket import inet_aton +import struct + +import apt_pkg + +from charmhelpers.core.hookenv import ( + config, + related_units, + relation_get, + relation_ids, + status_set, + open_port, +) +from charmhelpers.core.templating import render + +from common_utils import ( + get_ip, + decode_cert, + save_file, + check_run_prerequisites, + run_container, + json_loads, +) + +apt_pkg.init() +config = config() + + +CONTAINER_NAME = "contrail-analytics" +CONFIG_NAME = "analytics" +SERVICES_TO_CHECK = ["contrail-collector", "contrail-analytics-api"] + + +def controller_ctx(): + """Get the ipaddress of all contrail control nodes""" + auth_mode = config.get("auth_mode") + if auth_mode is None: + # NOTE: auth_mode must be transmitted by controller + return {} + + controller_ip_list = [] + for rid in relation_ids("contrail-analytics"): + for unit in related_units(rid): + if unit.startswith("contrail-controller"): + ip = relation_get("private-address", unit, rid) + controller_ip_list.append(ip) + sort_key = lambda ip: struct.unpack("!L", inet_aton(ip))[0] + controller_ip_list = sorted(controller_ip_list, key=sort_key) + return { + "auth_mode": auth_mode, + "controller_servers": controller_ip_list, + } + + +def analytics_ctx(): + """Get the ipaddress of all analytics control nodes""" + analytics_ip_list = [] + for rid in relation_ids("analytics-cluster"): + for unit in related_units(rid): + ip = relation_get("private-address", unit, rid) + analytics_ip_list.append(ip) + # add it's own ip address + analytics_ip_list.append(get_ip()) + sort_key = lambda ip: struct.unpack("!L", inet_aton(ip))[0] + analytics_ip_list = sorted(analytics_ip_list, key=sort_key) + return {"analytics_servers": analytics_ip_list} + + +def analyticsdb_ctx(): + """Get the ipaddress of all contrail analyticsdb nodes""" + analyticsdb_ip_list = [] + for rid in relation_ids("contrail-analyticsdb"): + for unit in related_units(rid): + ip = relation_get("private-address", unit, rid) + analyticsdb_ip_list.append(ip) + + sort_key = lambda ip: struct.unpack("!L", inet_aton(ip))[0] + analyticsdb_ip_list = sorted(analyticsdb_ip_list, key=sort_key) + return {"analyticsdb_servers": analyticsdb_ip_list} + + +def get_context(): + ctx = {} + ctx.update(json_loads(config.get("orchestrator_info"), dict())) + + ssl_ca = decode_cert("ssl_ca") + ctx["ssl_ca"] = ssl_ca + ctx["ssl_cert"] = decode_cert("ssl_cert") + ctx["ssl_key"] = decode_cert("ssl_key") + ctx["ssl_enabled"] = (ssl_ca is not None and len(ssl_ca) > 0) + + ctx["db_user"] = config.get("db_user") + ctx["db_password"] = config.get("db_password") + + ctx["rabbitmq_user"] = config.get("rabbitmq_user") + ctx["rabbitmq_password"] = config.get("rabbitmq_password") + ctx["rabbitmq_vhost"] = config.get("rabbitmq_vhost") + + ctx.update(controller_ctx()) + ctx.update(analytics_ctx()) + ctx.update(analyticsdb_ctx()) + ctx.update(json_loads(config.get("auth_info"), dict())) + return ctx + + +def render_config(ctx=None): + if not ctx: + ctx = get_context() + + # NOTE: store files in default paths cause no way to pass this path to + # some of components (sandesh) + ssl_ca = ctx["ssl_ca"] + save_file("/etc/contrailctl/ssl/ca-cert.pem", ssl_ca) + ssl_cert = ctx["ssl_cert"] + save_file("/etc/contrailctl/ssl/server.pem", ssl_cert) + ssl_key = ctx["ssl_key"] + save_file("/etc/contrailctl/ssl/server-privkey.pem", ssl_key) + + render("analytics.conf", "/etc/contrailctl/analytics.conf", ctx) + + +def update_charm_status(update_config=True): + update_config_func = render_config if update_config else None + result = check_run_prerequisites(CONTAINER_NAME, CONFIG_NAME, + update_config_func, SERVICES_TO_CHECK) + if not result: + return + + ctx = get_context() + missing_relations = [] + if not ctx.get("controller_servers"): + missing_relations.append("contrail-controller") + if not ctx.get("analyticsdb_servers"): + missing_relations.append("contrail-analyticsdb") + if missing_relations: + status_set('blocked', + 'Missing relations: ' + ', '.join(missing_relations)) + return + if not ctx.get("cloud_orchestrator"): + status_set('blocked', + 'Missing cloud_orchestrator info in relation ' + 'with contrail-controller.') + return + if not ctx.get("keystone_ip"): + status_set('blocked', + 'Missing auth info in relation with contrail-controller.') + return + if not ctx.get("db_user"): + # NOTE: Charms don't allow to deploy cassandra in AllowAll mode + status_set('blocked', + 'Missing DB user/password info in ' + 'relation with contrail-controller.') + if not ctx.get("rabbitmq_user"): + # NOTE: Charms don't allow to deploy rabbitmq with guest access + status_set('blocked', + 'Missing rabbitmq user/password info in ' + 'relation with contrail-controller.') + # TODO: what should happens if relation departed? + + render_config(ctx) + open_port(8081, "TCP") + + run_container(CONTAINER_NAME, "contrail-analytics") diff --git a/contrail-analytics/hooks/docker_utils.py b/contrail-analytics/hooks/docker_utils.py new file mode 100644 index 0000000..92605f0 --- /dev/null +++ b/contrail-analytics/hooks/docker_utils.py @@ -0,0 +1,185 @@ +import functools +from time import sleep, time + +from subprocess import ( + CalledProcessError, + check_call, + check_output +) +from charmhelpers.core.hookenv import ( + resource_get, + config, + log, + ERROR, +) + + +config = config() + + +DOCKER_PACKAGES = ["docker.engine"] +DOCKER_CLI = "/usr/bin/docker" + + +def retry(f=None, timeout=10, delay=2): + """Retry decorator. + + Provides a decorator that can be used to retry a function if it raises + an exception. + + :param timeout: timeout in seconds (default 10) + :param delay: retry delay in seconds (default 2) + + Examples:: + + # retry fetch_url function + @retry + def fetch_url(): + # fetch url + + # retry fetch_url function for 60 secs + @retry(timeout=60) + def fetch_url(): + # fetch url + """ + if not f: + return functools.partial(retry, timeout=timeout, delay=delay) + + @functools.wraps(f) + def func(*args, **kwargs): + start = time() + error = None + while True: + try: + return f(*args, **kwargs) + except Exception as e: + error = e + elapsed = time() - start + if elapsed >= timeout: + raise error + remaining = timeout - elapsed + sleep(delay if delay <= remaining else remaining) + return func + + +# NOTE: this code assumes that name of container is the part of the +# name of docker image + +def add_docker_repo(): + import platform + cmd = 'curl -fsSL https://apt.dockerproject.org/gpg | sudo apt-key add -' + check_output(cmd, shell=True) + dist = platform.linux_distribution()[2].strip() + cmd = "add-apt-repository " + \ + "\"deb https://apt.dockerproject.org/repo/ " + \ + "ubuntu-%s " % (dist) + \ + "main\"" + check_output(cmd, shell=True) + + +def is_container_launched(name): + # NOTE: 'paused' state is not getting into account if someone paused it + # NOTE: assume that this cmd is the same as inspect of state: + # [DOCKER_CLI, "inspect", "-f", "{{.State.Running}}", name] + cmd = DOCKER_CLI + ' ps | grep -w ' + name + try: + check_output(cmd, shell=True) + return True + except CalledProcessError: + return False + + +def is_container_present(name): + cmd = DOCKER_CLI + ' ps --all | grep -w ' + name + try: + check_output(cmd, shell=True) + return True + except CalledProcessError: + return False + + +def dpkg_version(name, pkg): + try: + return check_output([DOCKER_CLI, + "exec", + name, + "dpkg-query", + "-f", "${Version}\\n", "-W", pkg]).rstrip() + except CalledProcessError: + return None + + +def load_docker_image(name): + img_path = resource_get(name) + if not img_path: + return None + image_id = get_docker_image_id(name) + if image_id: + # remove previous image + check_call([DOCKER_CLI, "rmi", image_id]) + check_call([DOCKER_CLI, "load", "-i", img_path]) + return get_docker_image_id(name) + + +def get_docker_image_id(name): + try: + output = check_output(DOCKER_CLI + ' images | grep -w ' + name, + shell=True) + except CalledProcessError: + return None + output = output.decode('UTF-8').split('\n') + for line in output: + parts = line.split() + if name in parts[0]: + return parts[2].strip() + return None + + +def launch_docker_image(name, additional_args=[]): + image_id = get_docker_image_id(name) + if not image_id: + log(name + " docker image is not available", level=ERROR) + return + + orchestrator = config.get("cloud_orchestrator") + args = [DOCKER_CLI, + "run", + "--net=host", + "--cap-add=AUDIT_WRITE", + "--privileged", + "--restart=unless-stopped", + "--env='CLOUD_ORCHESTRATOR=%s'" % (orchestrator), + "--volume=/etc/contrailctl:/etc/contrailctl", + "--name=%s" % name] + args.extend(additional_args) + args.extend(["-itd", image_id]) + log("Run container with cmd: " + ' '.join(args)) + check_call(args) + + +def docker_cp(name, src, dst): + check_call([DOCKER_CLI, "cp", name + ":" + src, dst]) + + +def docker_exec(name, cmd): + cli = [DOCKER_CLI, "exec", name] + if isinstance(cmd, list): + cli.extend(cmd) + else: + cli.append(cmd) + output = check_output(cli) + return output.decode('UTF-8') + + +@retry(timeout=32, delay=10) +def apply_config_in_container(name, cfg_name): + try: + cmd = (DOCKER_CLI + ' exec ' + name + ' contrailctl config sync -v' + + ' -c ' + cfg_name) + check_call(cmd, shell=True) + return True + except CalledProcessError as e: + if e.returncode == 137: + log("Container was restarted. " + str(e.output), level=ERROR) + return False + raise diff --git a/contrail-analytics/hooks/http-services-relation-joined b/contrail-analytics/hooks/http-services-relation-joined new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/http-services-relation-joined @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/install b/contrail-analytics/hooks/install new file mode 100755 index 0000000..29ff689 --- /dev/null +++ b/contrail-analytics/hooks/install @@ -0,0 +1,20 @@ +#!/bin/bash +# Wrapper to deal with newer Ubuntu versions that don't have py2 installed +# by default. + +declare -a DEPS=('apt' 'netaddr' 'netifaces' 'pip' 'yaml' 'dnspython') + +check_and_install() { + pkg="${1}-${2}" + if ! dpkg -s ${pkg} 2>&1 > /dev/null; then + apt-get -y install ${pkg} + fi +} + +PYTHON="python" + +for dep in ${DEPS[@]}; do + check_and_install ${PYTHON} ${dep} +done + +exec ./hooks/install.real diff --git a/contrail-analytics/hooks/install.real b/contrail-analytics/hooks/install.real new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/install.real @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/update-status b/contrail-analytics/hooks/update-status new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/update-status @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/hooks/upgrade-charm b/contrail-analytics/hooks/upgrade-charm new file mode 120000 index 0000000..f633cfc --- /dev/null +++ b/contrail-analytics/hooks/upgrade-charm @@ -0,0 +1 @@ +contrail_analytics_hooks.py
\ No newline at end of file diff --git a/contrail-analytics/icon.svg b/contrail-analytics/icon.svg new file mode 100644 index 0000000..6f77c1a --- /dev/null +++ b/contrail-analytics/icon.svg @@ -0,0 +1,309 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- Created with Inkscape (http://www.inkscape.org/) --> + +<svg + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:cc="http://creativecommons.org/ns#" + xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:svg="http://www.w3.org/2000/svg" + xmlns="http://www.w3.org/2000/svg" + xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd" + xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape" + width="96" + height="96" + id="svg6517" + version="1.1" + inkscape:version="0.91 r13725" + sodipodi:docname="icon.svg"> + <defs + id="defs6519"> + <linearGradient + id="Background"> + <stop + id="stop4178" + offset="0" + style="stop-color:#b8b8b8;stop-opacity:1" /> + <stop + id="stop4180" + offset="1" + style="stop-color:#c9c9c9;stop-opacity:1" /> + </linearGradient> + <filter + style="color-interpolation-filters:sRGB;" + inkscape:label="Inner Shadow" + id="filter1121"> + <feFlood + flood-opacity="0.59999999999999998" + flood-color="rgb(0,0,0)" + result="flood" + id="feFlood1123" /> + <feComposite + in="flood" + in2="SourceGraphic" + operator="out" + result="composite1" + id="feComposite1125" /> + <feGaussianBlur + in="composite1" + stdDeviation="1" + result="blur" + id="feGaussianBlur1127" /> + <feOffset + dx="0" + dy="2" + result="offset" + id="feOffset1129" /> + <feComposite + in="offset" + in2="SourceGraphic" + operator="atop" + result="composite2" + id="feComposite1131" /> + </filter> + <filter + style="color-interpolation-filters:sRGB;" + inkscape:label="Drop Shadow" + id="filter950"> + <feFlood + flood-opacity="0.25" + flood-color="rgb(0,0,0)" + result="flood" + id="feFlood952" /> + <feComposite + in="flood" + in2="SourceGraphic" + operator="in" + result="composite1" + id="feComposite954" /> + <feGaussianBlur + in="composite1" + stdDeviation="1" + result="blur" + id="feGaussianBlur956" /> + <feOffset + dx="0" + dy="1" + result="offset" + id="feOffset958" /> + <feComposite + in="SourceGraphic" + in2="offset" + operator="over" + result="composite2" + id="feComposite960" /> + </filter> + <clipPath + clipPathUnits="userSpaceOnUse" + id="clipPath873"> + <g + transform="matrix(0,-0.66666667,0.66604479,0,-258.25992,677.00001)" + id="g875" + inkscape:label="Layer 1" + style="fill:#ff00ff;fill-opacity:1;stroke:none;display:inline"> + <path + style="fill:#ff00ff;fill-opacity:1;stroke:none;display:inline" + d="m 46.702703,898.22775 50.594594,0 C 138.16216,898.22775 144,904.06497 144,944.92583 l 0,50.73846 c 0,40.86071 -5.83784,46.69791 -46.702703,46.69791 l -50.594594,0 C 5.8378378,1042.3622 0,1036.525 0,995.66429 L 0,944.92583 C 0,904.06497 5.8378378,898.22775 46.702703,898.22775 Z" + id="path877" + inkscape:connector-curvature="0" + sodipodi:nodetypes="sssssssss" /> + </g> + </clipPath> + <filter + inkscape:collect="always" + id="filter891" + inkscape:label="Badge Shadow"> + <feGaussianBlur + inkscape:collect="always" + stdDeviation="0.71999962" + id="feGaussianBlur893" /> + </filter> + </defs> + <sodipodi:namedview + id="base" + pagecolor="#ffffff" + bordercolor="#666666" + borderopacity="1.0" + inkscape:pageopacity="0.0" + inkscape:pageshadow="2" + inkscape:zoom="4.0745362" + inkscape:cx="48.413329" + inkscape:cy="49.018169" + inkscape:document-units="px" + inkscape:current-layer="layer1" + showgrid="true" + fit-margin-top="0" + fit-margin-left="0" + fit-margin-right="0" + fit-margin-bottom="0" + inkscape:window-width="1920" + inkscape:window-height="1025" + inkscape:window-x="0" + inkscape:window-y="27" + inkscape:window-maximized="1" + showborder="true" + showguides="true" + inkscape:guide-bbox="true" + inkscape:showpageshadow="false"> + <inkscape:grid + type="xygrid" + id="grid821" /> + <sodipodi:guide + orientation="1,0" + position="16,48" + id="guide823" /> + <sodipodi:guide + orientation="0,1" + position="64,80" + id="guide825" /> + <sodipodi:guide + orientation="1,0" + position="80,40" + id="guide827" /> + <sodipodi:guide + orientation="0,1" + position="64,16" + id="guide829" /> + </sodipodi:namedview> + <metadata + id="metadata6522"> + <rdf:RDF> + <cc:Work + rdf:about=""> + <dc:format>image/svg+xml</dc:format> + <dc:type + rdf:resource="http://purl.org/dc/dcmitype/StillImage" /> + <dc:title></dc:title> + </cc:Work> + </rdf:RDF> + </metadata> + <g + inkscape:label="BACKGROUND" + inkscape:groupmode="layer" + id="layer1" + transform="translate(268,-635.29076)" + style="display:inline"> + <path + style="fill:#ebebeb;fill-opacity:1;stroke:none;display:inline;filter:url(#filter1121)" + d="m -268,700.15563 0,-33.72973 c 0,-27.24324 3.88785,-31.13513 31.10302,-31.13513 l 33.79408,0 c 27.21507,0 31.1029,3.89189 31.1029,31.13513 l 0,33.72973 c 0,27.24325 -3.88783,31.13514 -31.1029,31.13514 l -33.79408,0 C -264.11215,731.29077 -268,727.39888 -268,700.15563 Z" + id="path6455" + inkscape:connector-curvature="0" + sodipodi:nodetypes="sssssssss" /> + </g> + <g + inkscape:groupmode="layer" + id="layer3" + inkscape:label="PLACE YOUR PICTOGRAM HERE" + style="display:inline"> + <g + style="display:inline" + transform="matrix(0.30759127,0,0,0.30759127,8.28218,8.97257)" + id="g3732"> + <path + style="fill:#a3cfe8" + d="M 95,165.62616 C 84.317392,162.68522 76.316695,156.3432 71.320441,146.85577 68.731857,141.94027 68.5,140.61329 68.5,130.71353 c 0,-11.83269 0.397793,-12.66977 6.034392,-12.69822 C 78.926707,117.99315 81,121.97863 81,130.44413 c 0,9.5666 3.34886,15.50194 11.662711,20.67036 3.651393,2.26995 4.798754,2.40131 23.683989,2.71173 l 19.8467,0.32623 -0.71218,2.17377 c -0.91082,2.78009 -0.90418,5.58369 0.0199,8.42378 l 0.73211,2.25 -18.36663,-0.0675 C 106.56201,166.89096 97.76974,166.38867 95,165.62616 Z m 46.00868,-0.11571 c -1.77687,-2.14099 -1.82625,-7.82041 -0.0862,-9.917 1.07681,-1.29747 3.57513,-1.59374 13.45,-1.595 9.54779,-0.001 12.86912,-0.37349 15.61365,-1.75 9.3963,-4.71272 7.35301,-19.21115 -2.93942,-20.85698 -2.07398,-0.33164 -4.19534,-0.89289 -4.71413,-1.24723 -0.51879,-0.35433 -1.44954,-3.43526 -2.06833,-6.84652 -1.37797,-7.59639 -3.48916,-12.20669 -7.30276,-15.94738 -3.66382,-3.59378 -3.6595,-4.21104 0.0385,-5.50018 2.54055,-0.88564 3,-1.56686 3,-4.447985 0,-4.258462 1.35388,-4.297632 5.25974,-0.152175 4.55275,4.83203 8.57589,11.55276 10.42257,17.41111 1.15326,3.65858 2.26012,5.35908 3.72889,5.72883 3.21482,0.8093 9.54053,7.29049 11.64977,11.9361 2.26213,4.98232 2.53846,14.30356 0.56413,19.02881 -1.97355,4.72336 -7.28419,10.42159 -12.03042,12.90844 -3.50369,1.8358 -6.19345,2.20312 -18.636,2.54499 -12.76506,0.35072 -14.7134,0.19219 -15.95,-1.29783 z M 36.760565,161.75 c -3.478655,-4.56459 -7.187084,-12.21027 -9.336932,-19.25 -2.778434,-9.09804 -2.583706,-24.94034 0.417306,-33.95043 3.497444,-10.500559 9.898641,-21.56636 12.457102,-21.534693 0.661077,0.0082 2.925911,1.473635 5.032964,3.256562 l 3.831004,3.241685 -2.568452,5.113673 C 42.599304,106.57918 40.65102,115.46967 40.594928,126 c -0.0579,10.86969 1.439444,17.99787 5.535634,26.35262 1.578191,3.21895 2.85983,6.14395 2.848087,6.5 C 48.949775,159.72808 41.428955,165 40.208913,165 c -0.534344,0 -2.086101,-1.4625 -3.448348,-3.25 z m 175.995035,-0.0376 -3.7444,-3.21245 1.79249,-3 c 8.93434,-14.95294 9.53034,-38.50427 1.41338,-55.849827 l -3.07866,-6.578941 4.1278,-3.035616 C 215.5365,88.366027 217.71535,87 218.10811,87 c 1.50502,0 6.33619,6.757331 8.97827,12.55785 7.79191,17.10669 7.87368,37.40315 0.21328,52.94215 -2.91602,5.91511 -7.82715,12.49548 -9.29966,12.46052 -0.825,-0.0196 -3.18498,-1.48122 -5.2444,-3.24807 z M 81.482645,115.96644 c -1.483807,-2.86937 -1.949857,-3.10137 -5.058516,-2.51818 -4.663007,0.87478 -4.493442,-0.95188 0.628511,-6.77072 5.256509,-5.97171 14.327595,-10.460488 22.924736,-11.34418 4.557714,-0.468483 7.786604,-1.496091 10.894994,-3.467375 10.33444,-6.553906 24.98246,-8.287165 35.62763,-4.215718 4.82222,1.84435 5,2.051462 5,5.824988 0,3.32368 -0.46902,4.186565 -3.11582,5.732379 -2.93452,1.713856 -3.47765,1.727036 -9.3345,0.226582 -5.19732,-1.331492 -7.06708,-1.394156 -11.38418,-0.381538 -6.35168,1.489842 -8.08332,2.337822 -13.18203,6.455152 -3.63495,2.93531 -4.49954,3.19704 -9.10062,2.75494 -6.189167,-0.59471 -12.218344,1.78693 -18.196739,7.18806 l -4.06908,3.67616 -1.634386,-3.16055 z" + id="path3746" + inkscape:connector-curvature="0" /> + <path + style="fill:#9a9a9c" + d="m 93.286039,164.54925 c -16.494387,-5.15489 -26.958648,-21.00658 -24.875196,-37.68196 0.843223,-6.74892 1.329136,-7.48226 5.337762,-8.05574 4.602358,-0.65842 6.634722,2.66079 6.356138,10.38072 -0.355642,9.8553 5.007342,19.02839 13.395257,22.91187 3.449975,1.59728 6.65053,1.85496 23.27568,1.8739 l 19.27568,0.022 -1.5223,2.9438 c -1.13702,2.19876 -1.27006,3.60722 -0.52568,5.5651 0.54814,1.44171 0.99662,2.817 0.99662,3.0562 0,1.13237 -37.784447,0.21221 -41.713961,-1.01585 z M 140.3757,163.25 c -0.75749,-2.06167 -0.6343,-3.56348 0.49217,-6 l 1.50255,-3.25 12.9105,0 c 14.6294,0 17.5288,-0.97189 20.29597,-6.80328 3.45454,-7.27989 -1.32251,-15.43619 -9.78395,-16.70506 l -4.53221,-0.67965 -0.51854,-5.71858 c -0.55357,-6.10485 -4.15117,-14.35103 -7.6341,-17.49842 -2.70447,-2.44391 -2.6528,-3.02579 0.39191,-4.41306 1.58875,-0.72388 2.50558,-1.96702 2.51531,-3.410511 0.008,-1.249292 0.39216,-2.865775 0.85274,-3.592185 C 158.67512,92.329247 172,111.55317 172,117.01025 c 0,0.94756 2.19487,3.0552 4.99312,4.79469 16.07824,9.99478 15.53196,32.74917 -0.99499,41.44506 -5.0138,2.63808 -5.82451,2.75 -19.91928,2.75 l -14.69277,0 -1.01038,-2.75 z M 35.40716,159.29417 c -2.083023,-3.13821 -5.109308,-9.54119 -6.725077,-14.22886 -2.485242,-7.21018 -2.938617,-10.06664 -2.943307,-18.54417 -0.0036,-6.59373 0.591734,-12.07325 1.74079,-16.02114 2.125307,-7.30206 7.833992,-18.506493 10.893586,-21.380833 l 2.245692,-2.109718 4.114129,3.025565 4.114129,3.025564 -2.940589,6.48533 c -7.687874,16.955242 -7.684823,36.645922 0.0082,53.085582 l 2.95122,6.30662 -3.826883,3.03094 C 42.934289,163.63607 40.758205,165 40.203333,165 c -0.554872,0 -2.71315,-2.56762 -4.796173,-5.70583 z m 178.33231,2.91881 c -4.12643,-2.97696 -4.12127,-2.77305 -0.30142,-11.89827 C 216.73845,142.43037 218,135.70645 218,126 c 0,-9.70412 -1.26117,-16.4284 -4.56034,-24.31471 -1.42316,-3.401907 -2.66678,-6.795138 -2.76361,-7.540509 -0.0968,-0.74537 1.55376,-2.77037 3.66797,-4.5 L 218.18803,86.5 l 2.46357,3 c 10.21069,12.43401 14.79345,33.98475 10.72523,50.43611 -2.37412,9.60065 -10.56942,25.165 -13.17772,25.02687 -0.38451,-0.0204 -2.39135,-1.25787 -4.45964,-2.75 z M 81.841186,115.55079 c -0.878315,-1.9277 -1.99166,-2.51327 -5.228562,-2.75 L 72.5,112.5 77.225927,107.42203 C 83.456988,100.72681 89.946931,97.312559 99.091117,95.919125 103.166,95.298175 107.175,94.376154 108,93.87019 c 0.825,-0.505965 4.40457,-2.344245 7.95461,-4.085068 8.22915,-4.035307 19.81365,-4.987772 28.27907,-2.325071 7.55962,2.37779 7.79351,2.597566 7.12811,6.697941 C 150.57502,99.006294 146.1878,101.20891 141,99.36016 132.99683,96.508113 122.06502,98.684599 115.29736,104.47747 111.53712,107.6961 110.64067,108 104.90676,108 97.846719,108 92.517648,110.09663 87.188282,114.97101 85.366837,116.63695 83.669689,118 83.416843,118 c -0.252846,0 -0.961892,-1.10215 -1.575657,-2.44921 z" + id="path3744" + inkscape:connector-curvature="0" /> + <path + style="fill:#50a1d2" + d="m 93.286039,164.54925 c -16.494387,-5.15489 -26.958648,-21.00658 -24.875196,-37.68196 0.843223,-6.74892 1.329136,-7.48226 5.337762,-8.05574 4.602358,-0.65842 6.634722,2.66079 6.356138,10.38072 -0.355642,9.8553 5.007342,19.02839 13.395257,22.91187 3.449975,1.59728 6.65053,1.85496 23.27568,1.8739 l 19.27568,0.022 -1.5223,2.9438 c -1.13702,2.19876 -1.27006,3.60722 -0.52568,5.5651 0.54814,1.44171 0.99662,2.817 0.99662,3.0562 0,1.13237 -37.784447,0.21221 -41.713961,-1.01585 z M 140.3757,163.25 c -0.75749,-2.06167 -0.6343,-3.56348 0.49217,-6 l 1.50255,-3.25 12.9105,0 c 14.6294,0 17.5288,-0.97189 20.29597,-6.80328 3.45454,-7.27989 -1.32251,-15.43619 -9.78395,-16.70506 l -4.53221,-0.67965 -0.51854,-5.71858 c -0.55357,-6.10485 -4.15117,-14.35103 -7.6341,-17.49842 -2.70447,-2.44391 -2.6528,-3.02579 0.39191,-4.41306 1.58875,-0.72388 2.50558,-1.96702 2.51531,-3.410511 0.008,-1.249292 0.39216,-2.865775 0.85274,-3.592185 C 158.67512,92.329247 172,111.55317 172,117.01025 c 0,0.94756 2.19487,3.0552 4.99312,4.79469 16.07824,9.99478 15.53196,32.74917 -0.99499,41.44506 -5.0138,2.63808 -5.82451,2.75 -19.91928,2.75 l -14.69277,0 -1.01038,-2.75 z M 36.924699,160.79198 C 33.485946,156.10457 30.687068,150.24942 28.180767,142.5 c -2.22154,-6.86895 -2.214797,-26.11727 0.01161,-33.13024 2.21057,-6.96308 6.348289,-15.18965 9.611074,-19.108624 L 40.5,87.022271 l 3.875471,3.282759 3.875472,3.282758 -2.18708,4.287031 c -7.653476,15.002051 -8.071995,38.329351 -0.968739,53.995241 3.168854,6.98876 3.078371,7.44609 -2.21963,11.2186 l -2.802135,1.99529 -3.14866,-4.29197 z m 177.289621,1.13424 -4.17969,-3.07377 1.95557,-3.83324 c 5.55817,-10.89491 7.78283,-24.62144 6.0729,-37.4708 -0.61859,-4.64838 -1.81396,-10.16088 -2.65638,-12.25 -1.54072,-3.82085 -4.3711,-10.259911 -5.02182,-11.424556 -0.6119,-1.095168 7.44846,-6.09488 8.63936,-5.35886 2.42142,1.496519 8.05598,11.676956 10.60291,19.157176 3.82818,11.24317 3.81121,25.44418 -0.044,36.82783 -2.07525,6.12777 -9.78971,20.5 -11.00362,20.5 -0.10204,0 -2.06639,-1.3832 -4.36522,-3.07378 z M 81.841186,115.55079 c -0.878315,-1.9277 -1.99166,-2.51327 -5.228562,-2.75 L 72.5,112.5 77.225927,107.42203 C 83.456988,100.72681 89.946931,97.312559 99.091117,95.919125 103.166,95.298175 107.175,94.376154 108,93.87019 c 0.825,-0.505965 4.40457,-2.344245 7.95461,-4.085068 8.22915,-4.035307 19.81365,-4.987772 28.27907,-2.325071 7.55962,2.37779 7.79351,2.597566 7.12811,6.697941 C 150.57502,99.006294 146.1878,101.20891 141,99.36016 132.99683,96.508113 122.06502,98.684599 115.29736,104.47747 111.53712,107.6961 110.64067,108 104.90676,108 97.846719,108 92.517648,110.09663 87.188282,114.97101 85.366837,116.63695 83.669689,118 83.416843,118 c -0.252846,0 -0.961892,-1.10215 -1.575657,-2.44921 z" + id="path3742" + inkscape:connector-curvature="0" /> + <path + style="fill:#258bc8" + d="m 140.94241,163.34852 c -0.60534,-1.59216 -0.6633,-3.68963 -0.14507,-5.25 0.8603,-2.5903 0.90545,-2.60011 14.28284,-3.09996 7.93908,-0.29664 14.30706,-1.00877 15.59227,-1.74367 10.44037,-5.96999 7.38458,-21.04866 -4.67245,-23.05598 l -4.5,-0.74919 -0.58702,-5.97486 c -0.62455,-6.35693 -3.09323,-12.09225 -7.29978,-16.95905 l -2.57934,-2.98419 2.20484,-0.81562 c 2.73303,-1.01102 3.71477,-2.49335 3.78569,-5.716 0.0511,-2.322172 0.38375,-2.144343 4.67651,2.5 4.32664,4.681 10.2991,15.64731 10.2991,18.91066 0,0.80001 0.94975,1.756 2.11054,2.12443 3.25146,1.03197 9.8171,7.40275 11.96188,11.60686 2.54215,4.98304 2.56222,14.86412 0.0414,20.41386 -2.26808,4.99343 -8.79666,10.73297 -13.97231,12.28363 C 170.01108,165.47775 162.34653,166 155.10923,166 l -13.15873,0 -1.00809,-2.65148 z M 36.924699,160.79198 C 33.485946,156.10457 30.687068,150.24942 28.180767,142.5 c -2.22154,-6.86895 -2.214797,-26.11727 0.01161,-33.13024 2.21057,-6.96308 6.348289,-15.18965 9.611074,-19.108624 L 40.5,87.022271 l 3.875471,3.282759 3.875472,3.282758 -2.18708,4.287031 c -7.653476,15.002051 -8.071995,38.329351 -0.968739,53.995241 3.168854,6.98876 3.078371,7.44609 -2.21963,11.2186 l -2.802135,1.99529 -3.14866,-4.29197 z m 177.289621,1.13424 -4.17969,-3.07377 1.95557,-3.83324 c 5.55817,-10.89491 7.78283,-24.62144 6.0729,-37.4708 -0.61859,-4.64838 -1.81396,-10.16088 -2.65638,-12.25 -1.54072,-3.82085 -4.3711,-10.259911 -5.02182,-11.424556 -0.6119,-1.095168 7.44846,-6.09488 8.63936,-5.35886 2.42142,1.496519 8.05598,11.676956 10.60291,19.157176 3.82818,11.24317 3.81121,25.44418 -0.044,36.82783 -2.07525,6.12777 -9.78971,20.5 -11.00362,20.5 -0.10204,0 -2.06639,-1.3832 -4.36522,-3.07378 z M 81.664567,115.0093 c -1.516672,-2.56752 -2.095101,-2.81369 -5.364599,-2.28313 l -3.66463,0.59469 2.22168,-3.12006 C 80.37626,102.44974 90.120126,97.000633 99.857357,96.219746 105.13094,95.796826 107.53051,95.01192 111.5,92.411404 c 10.08936,-6.609802 24.47284,-8.157994 35.30015,-3.799597 4.05392,1.631857 4.28296,1.935471 4,5.302479 -0.41543,4.943233 -3.85308,6.604794 -10.30411,4.980399 -9.07108,-2.284124 -18.26402,-0.195093 -26.41897,6.003525 -2.78485,2.11679 -4.55576,2.61322 -9.5,2.66311 -6.674981,0.0673 -12.069467,2.29808 -17.866999,7.38838 l -3.345536,2.93742 -1.699968,-2.87782 z" + id="path3740" + inkscape:connector-curvature="0" /> + <path + style="fill:#6c6d71" + d="M 36.924699,160.79198 C 33.485946,156.10457 30.687068,150.24942 28.180767,142.5 c -2.22154,-6.86895 -2.214797,-26.11727 0.01161,-33.13024 2.21057,-6.96308 6.348289,-15.18965 9.611074,-19.108624 L 40.5,87.022271 l 3.875471,3.282759 3.875472,3.282758 -2.18708,4.287031 c -7.653476,15.002051 -8.071995,38.329351 -0.968739,53.995241 3.168854,6.98876 3.078371,7.44609 -2.21963,11.2186 l -2.802135,1.99529 -3.14866,-4.29197 z m 177.289621,1.13424 -4.17969,-3.07377 1.95557,-3.83324 c 5.55817,-10.89491 7.78283,-24.62144 6.0729,-37.4708 -0.61859,-4.64838 -1.81396,-10.16088 -2.65638,-12.25 -1.54072,-3.82085 -4.3711,-10.259911 -5.02182,-11.424556 -0.6119,-1.095168 7.44846,-6.09488 8.63936,-5.35886 2.42142,1.496519 8.05598,11.676956 10.60291,19.157176 3.82818,11.24317 3.81121,25.44418 -0.044,36.82783 -2.07525,6.12777 -9.78971,20.5 -11.00362,20.5 -0.10204,0 -2.06639,-1.3832 -4.36522,-3.07378 z M 81.778822,114.41391 c -0.987352,-2.167 -1.713119,-2.52365 -4.478561,-2.2008 C 75.485117,112.42502 74,112.28006 74,111.89098 c 0,-0.38909 2.038348,-2.80473 4.529662,-5.36811 5.687016,-5.85151 13.385461,-9.421936 22.389748,-10.384041 4.19603,-0.448345 7.72119,-1.408591 8.81929,-2.402352 1.0061,-0.910509 4.51398,-2.848867 7.79529,-4.307463 11.5167,-5.119364 33.48865,-2.808232 33.4507,3.51853 -0.03,5.002939 -4.29101,7.838526 -9.20479,6.125573 -1.69309,-0.590214 -6.0487,-1.063234 -9.67912,-1.051155 -7.46196,0.02483 -12.78325,2.004318 -18.21979,6.777668 -3.02474,2.65576 -4.03125,2.9899 -7.5746,2.51464 -5.45614,-0.73182 -12.97717,1.85611 -18.074646,6.21936 -2.22732,1.9065 -4.325286,3.46637 -4.662147,3.46637 -0.336861,0 -1.14271,-1.16374 -1.790775,-2.58609 z" + id="path3738" + inkscape:connector-curvature="0" /> + <path + style="fill:#0076c2" + d="m 81.778822,114.41391 c -0.987352,-2.167 -1.713119,-2.52365 -4.478561,-2.2008 C 75.485117,112.42502 74,112.28006 74,111.89098 c 0,-0.38909 2.038348,-2.80473 4.529662,-5.36811 5.687016,-5.85151 13.385461,-9.421936 22.389748,-10.384041 4.19603,-0.448345 7.72119,-1.408591 8.81929,-2.402352 1.0061,-0.910509 4.51398,-2.848867 7.79529,-4.307463 11.5167,-5.119364 33.48865,-2.808232 33.4507,3.51853 -0.03,5.002939 -4.29101,7.838526 -9.20479,6.125573 -1.69309,-0.590214 -6.0487,-1.063234 -9.67912,-1.051155 -7.46196,0.02483 -12.78325,2.004318 -18.21979,6.777668 -3.02474,2.65576 -4.03125,2.9899 -7.5746,2.51464 -5.45614,-0.73182 -12.97717,1.85611 -18.074646,6.21936 -2.22732,1.9065 -4.325286,3.46637 -4.662147,3.46637 -0.336861,0 -1.14271,-1.16374 -1.790775,-2.58609 z" + id="path3736" + inkscape:connector-curvature="0" /> + <path + style="fill:#0275bc" + d="m 84,115.94098 c 0,-0.58246 -0.519529,-0.73793 -1.154508,-0.34549 -0.691266,0.42723 -0.883989,0.27582 -0.48031,-0.37735 0.370809,-0.59998 1.542397,-1.02548 2.603528,-0.94554 1.457446,0.10978 1.667267,0.4611 0.857865,1.43636 C 84.525185,117.27704 84,117.34375 84,115.94098 Z m 0.09671,-3.86005 c -1.011759,-0.64056 -0.689769,-0.84554 1.15404,-0.73469 1.406534,0.0846 2.348958,0.49126 2.094276,0.90376 -0.60193,0.97493 -1.516575,0.92732 -3.248316,-0.16907 z m 6.3078,-0.92642 c 0.398903,-0.64544 0.136326,-1.16792 -0.595491,-1.18492 -0.765174,-0.0178 -0.541923,-0.47628 0.537358,-1.10362 1.338377,-0.77794 2.163776,-0.75328 3,0.0896 0.874885,0.8819 0.691151,0.98669 -0.76042,0.43369 -1.280472,-0.48782 -1.688838,-0.3648 -1.233688,0.37165 0.374196,0.60547 0.153488,1.42647 -0.490464,1.82445 -0.731227,0.45192 -0.902922,0.29014 -0.457295,-0.4309 z M 78.5,109.91171 l -3,-0.7763 3.217276,0.16818 c 2.186877,0.11431 3.688589,-0.46785 4.688882,-1.81771 1.457369,-1.96667 1.489127,-1.96706 3.282724,-0.0406 1.583464,1.70072 1.591856,1.78019 0.06676,0.63224 -1.483392,-1.11656 -2.007002,-1.0195 -3.5,0.64877 -1.381497,1.54369 -2.394984,1.79632 -4.755647,1.18547 z M 78.5,107 c -0.60158,-0.97338 0.120084,-1.39478 1.85526,-1.08333 1.302991,0.23387 3.690445,-2.0337 3.117418,-2.96088 -0.277916,-0.44968 0.02157,-1.14322 0.665519,-1.5412 0.731227,-0.45192 0.902922,-0.29014 0.457295,0.4309 -1.008441,1.63169 1.517118,1.38391 3.845638,-0.37729 1.067621,-0.80751 2.867621,-1.42334 4,-1.36852 2.027174,0.0981 2.02808,0.11053 0.05887,0.80463 -4.600356,1.62151 -9.243399,4.08158 -10.452051,5.53791 C 80.556518,108.23929 79.380215,108.42422 78.5,107 Z m 12.25,-0.66228 c 0.6875,-0.27741 1.8125,-0.27741 2.5,0 0.6875,0.27741 0.125,0.50439 -1.25,0.50439 -1.375,0 -1.9375,-0.22698 -1.25,-0.50439 z m -1.953895,-1.90746 c 1.232615,-0.86336 3.020243,-1.36556 3.972506,-1.116 1.314258,0.34442 1.203531,0.48168 -0.459594,0.56974 -1.205041,0.0638 -2.469098,0.566 -2.809017,1.116 -0.339919,0.55 -1.141604,1 -1.781523,1 -0.639919,0 -0.154987,-0.70638 1.077628,-1.56974 z m 12.467645,-0.14784 c 1.52006,-0.22986 3.77006,-0.22371 5,0.0136 1.22994,0.23736 -0.0138,0.42542 -2.76375,0.41792 -2.75,-0.008 -3.756313,-0.20172 -2.23625,-0.43157 z m 13.52519,-3.66627 c 1.62643,-1.858573 1.61751,-1.921032 -0.18038,-1.262823 -1.58361,0.579759 -1.69145,0.451477 -0.6626,-0.788214 0.96581,-1.163733 1.50975,-1.222146 2.54116,-0.272892 0.80101,0.737212 0.96515,1.63324 0.42127,2.299789 -0.49007,0.6006 -0.69137,1.29168 -0.44733,1.53571 0.24403,0.24404 -0.41735,0.44371 -1.46974,0.44371 -1.81559,0 -1.82594,-0.1 -0.20238,-1.95528 z m -13.35766,0.48689 c 1.8068,-0.70764 6.56872,-0.33535 6.56872,0.51354 0,0.21088 -1.9125,0.35179 -4.25,0.31313 -3.00669,-0.0497 -3.68502,-0.29156 -2.31872,-0.82667 z M 120,98.984687 c -1.33333,-0.875277 -1.33333,-1.094097 0,-1.969374 0.825,-0.541578 2.175,-0.939378 3,-0.883999 0.99463,0.06677 0.88566,0.259531 -0.32343,0.572152 -1.07213,0.27721 -1.60009,1.05346 -1.28138,1.883999 0.63873,1.664515 0.5666,1.685055 -1.39519,0.397222 z m 23.8125,0.332199 c 0.72187,-0.288871 1.58437,-0.253344 1.91667,0.07895 0.33229,0.332292 -0.25834,0.568641 -1.3125,0.52522 -1.16495,-0.04798 -1.4019,-0.284941 -0.60417,-0.604167 z M 100,98.073324 c 0,-0.509672 -0.7875,-1.132471 -1.75,-1.383998 -1.31691,-0.344145 -1.19317,-0.486031 0.5,-0.573325 1.2375,-0.0638 2.25,0.305488 2.25,0.820641 0,0.515152 1.4625,1.118136 3.25,1.339962 3.19982,0.397095 3.1921,0.405793 -0.5,0.563359 -2.0625,0.08802 -3.75,-0.256967 -3.75,-0.766639 z m 29.75,-0.79672 c 1.7875,-0.221826 4.7125,-0.221826 6.5,0 1.7875,0.221827 0.325,0.403322 -3.25,0.403322 -3.575,0 -5.0375,-0.181495 -3.25,-0.403322 z M 142.5,97 c -1.75921,-0.755957 -1.6618,-0.867892 0.80902,-0.929715 1.63221,-0.04084 2.5501,0.348653 2.19098,0.929715 -0.33992,0.55 -0.70398,0.968372 -0.80902,0.929715 C 144.58594,97.891058 143.6,97.472686 142.5,97 Z m -32.85536,-1.199796 c 0.45361,-0.715112 0.83163,-1.600204 0.84005,-1.966871 0.008,-0.366666 0.42496,-1.041666 0.92564,-1.5 0.52889,-0.484163 0.60891,-0.309578 0.19098,0.416667 -0.93393,1.62288 0.27843,1.533702 3.39869,-0.25 2.99559,-1.712435 4,-1.837986 4,-0.5 0,0.55 -0.56916,1 -1.26481,1 -0.69564,0 -2.98616,0.922592 -5.09004,2.050204 -2.18676,1.172033 -3.47198,1.493283 -3.00051,0.75 z M 147,95.559017 C 147,94.701558 147.45,94 148,94 c 0.55,0 1,0.423442 1,0.940983 0,0.517541 -0.45,1.219098 -1,1.559017 -0.55,0.339919 -1,-0.08352 -1,-0.940983 z M 116.5,95 c 0.33992,-0.55 1.04148,-1 1.55902,-1 0.51754,0 0.94098,0.45 0.94098,1 0,0.55 -0.70156,1 -1.55902,1 -0.85746,0 -1.2809,-0.45 -0.94098,-1 z m 8.5,0.185596 c 0,-1.012848 13.57404,-0.944893 14.59198,0.07305 C 139.99972,95.666391 136.88333,96 132.66667,96 128.45,96 125,95.633518 125,95.185596 Z M 150.15789,94 c 0,-1.375 0.22698,-1.9375 0.50439,-1.25 0.27741,0.6875 0.27741,1.8125 0,2.5 -0.27741,0.6875 -0.50439,0.125 -0.50439,-1.25 z M 120.75,93.337719 c 0.6875,-0.277412 1.8125,-0.277412 2.5,0 0.6875,0.277413 0.125,0.504386 -1.25,0.504386 -1.375,0 -1.9375,-0.226973 -1.25,-0.504386 z m 21.51903,-0.03071 c 0.97297,-0.253543 2.32297,-0.236869 3,0.03705 0.67703,0.273923 -0.11903,0.481368 -1.76903,0.460988 -1.65,-0.02038 -2.20394,-0.244498 -1.23097,-0.498042 z M 126,91.822487 c 0,-1.159476 11.18403,-0.998163 13,0.187505 1.04165,0.680102 -0.71538,0.92675 -5.75,0.807174 C 129.2625,92.722461 126,92.274855 126,91.822487 Z M 147,92 c 0,-0.55 0.45,-1 1,-1 0.55,0 1,0.45 1,1 0,0.55 -0.45,1 -1,1 -0.55,0 -1,-0.45 -1,-1 z m -22.5,-2.531662 c 5.25889,-1.588265 12.55323,-1.437163 18.5,0.383229 3.35111,1.025823 3.2873,1.051779 -1.5,0.610174 -8.02324,-0.740105 -13.71413,-0.773698 -18,-0.106252 -3.61325,0.562697 -3.51656,0.476921 1,-0.887151 z m -1.6875,-2.151452 c 0.72187,-0.288871 1.58437,-0.253344 1.91667,0.07895 0.33229,0.332292 -0.25834,0.568641 -1.3125,0.52522 -1.16495,-0.04798 -1.4019,-0.284941 -0.60417,-0.604167 z m 8.45653,-1.009877 c 0.97297,-0.253543 2.32297,-0.236869 3,0.03705 0.67703,0.273923 -0.11903,0.481368 -1.76903,0.460988 -1.65,-0.02038 -2.20394,-0.244498 -1.23097,-0.498042 z" + id="path3734" + inkscape:connector-curvature="0" /> + </g> + </g> + <g + inkscape:groupmode="layer" + id="layer2" + inkscape:label="BADGE" + style="display:none" + sodipodi:insensitive="true"> + <g + style="display:inline" + transform="translate(-340.00001,-581)" + id="g4394" + clip-path="none"> + <g + id="g855"> + <g + inkscape:groupmode="maskhelper" + id="g870" + clip-path="url(#clipPath873)" + style="opacity:0.6;filter:url(#filter891)"> + <path + transform="matrix(1.4999992,0,0,1.4999992,-29.999795,-237.54282)" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + sodipodi:ry="12" + sodipodi:rx="12" + sodipodi:cy="552.36218" + sodipodi:cx="252" + id="path844" + style="color:#000000;fill:#000000;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + sodipodi:type="arc" /> + </g> + <g + id="g862"> + <path + sodipodi:type="arc" + style="color:#000000;fill:#f5f5f5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + id="path4398" + sodipodi:cx="252" + sodipodi:cy="552.36218" + sodipodi:rx="12" + sodipodi:ry="12" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + transform="matrix(1.4999992,0,0,1.4999992,-29.999795,-238.54282)" /> + <path + transform="matrix(1.25,0,0,1.25,33,-100.45273)" + d="m 264,552.36218 a 12,12 0 0 1 -12,12 12,12 0 0 1 -12,-12 12,12 0 0 1 12,-12 12,12 0 0 1 12,12 z" + sodipodi:ry="12" + sodipodi:rx="12" + sodipodi:cy="552.36218" + sodipodi:cx="252" + id="path4400" + style="color:#000000;fill:#dd4814;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:4;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + sodipodi:type="arc" /> + <path + sodipodi:type="star" + style="color:#000000;fill:#f5f5f5;fill-opacity:1;fill-rule:nonzero;stroke:none;stroke-width:3;marker:none;visibility:visible;display:inline;overflow:visible;enable-background:accumulate" + id="path4459" + sodipodi:sides="5" + sodipodi:cx="666.19574" + sodipodi:cy="589.50385" + sodipodi:r1="7.2431178" + sodipodi:r2="4.3458705" + sodipodi:arg1="1.0471976" + sodipodi:arg2="1.6755161" + inkscape:flatsided="false" + inkscape:rounded="0.1" + inkscape:randomized="0" + d="m 669.8173,595.77657 c -0.39132,0.22593 -3.62645,-1.90343 -4.07583,-1.95066 -0.44938,-0.0472 -4.05653,1.36297 -4.39232,1.06062 -0.3358,-0.30235 0.68963,-4.03715 0.59569,-4.47913 -0.0939,-0.44198 -2.5498,-3.43681 -2.36602,-3.8496 0.18379,-0.41279 4.05267,-0.59166 4.44398,-0.81759 0.39132,-0.22593 2.48067,-3.48704 2.93005,-3.4398 0.44938,0.0472 1.81505,3.67147 2.15084,3.97382 0.3358,0.30236 4.08294,1.2817 4.17689,1.72369 0.0939,0.44198 -2.9309,2.86076 -3.11469,3.27355 -0.18379,0.41279 0.0427,4.27917 -0.34859,4.5051 z" + transform="matrix(1.511423,-0.16366377,0.16366377,1.511423,-755.37346,-191.93651)" /> + </g> + </g> + </g> + </g> +</svg> diff --git a/contrail-analytics/metadata.yaml b/contrail-analytics/metadata.yaml new file mode 100644 index 0000000..9ca4d98 --- /dev/null +++ b/contrail-analytics/metadata.yaml @@ -0,0 +1,25 @@ +name: contrail-analytics +summary: Contrail Analytics Node +maintainer: Andrey Pavlov <andrey.mp@gmail.com> +description: | + Contral analytics component within docker container. +series: + - xenial + - trusty +peers: + analytics-cluster: + interface: contrail-analytics-cluster +provides: + contrail-analytics: + interface: contrail-analytics + http-services: + interface: http +requires: + contrail-analyticsdb: + interface: contrail-analyticsdb +resources: + contrail-analytics: + type: file + filename: contrail-analytics.tar.gz + description: "Contrail analytics docker image" + diff --git a/contrail-analytics/templates/analytics.conf b/contrail-analytics/templates/analytics.conf new file mode 100644 index 0000000..b221852 --- /dev/null +++ b/contrail-analytics/templates/analytics.conf @@ -0,0 +1,110 @@ +[GLOBAL] +# Default log_level +#log_level = SYS_NOTICE + +cloud_orchestrator = {{ cloud_orchestrator }} + +controller_nodes = {{ controller_servers|join(',') }} +analyticsdb_nodes = {{ analyticsdb_servers|join(',') }} +analytics_nodes = {{ analytics_servers|join(',') }} + +#uve_partition_count = {{ uve_partition_count }} + +xmpp_auth_enable = {{ ssl_enabled }} +xmpp_dns_auth_enable = {{ ssl_enabled }} +sandesh_ssl_enable = {{ ssl_enabled }} +introspect_ssl_enable = {{ ssl_enabled }} + +analyticsdb_cassandra_user = {{ db_user }} +analyticsdb_cassandra_password = {{ db_password }} + +[KEYSTONE] +version = {{ keystone_api_suffix }} +ip = {{ keystone_ip }} +admin_port = 35357 +public_port = {{ keystone_public_port }} +auth_protocol = {{ keystone_protocol }} +admin_user = {{ keystone_admin_user }} +admin_password = {{ keystone_admin_password }} +admin_tenant = {{ keystone_admin_tenant }} +insecure = True + +[ANALYTICS_API] +# Introspect port for debug +#introspect_port = 8090 +# +# Listen address and port +#listen_port = 8081 +#listen_address = {{ listen_address }} +# +# Log file and log_level +#log_level = SYS_NOTICE +#log = /var/log/contrail/contrail-analytics-api.log +# + +# aaa_mode - RBAC configuration for analytics api +# no-auth - no authentication is performed and full access is granted to all +# cloud-admin - authentication is performed and only cloud-admin role has access - default cloud-admin role is "admin" +# rbac - authentication is performed and access granted based on role and configured rules +aaa_mode = {{ auth_mode }} + +[ALARM_GEN] +# Log file and log level +#log = /var/log/contrail/contrail-alarm-gen.log +#log_level = SYS_NOTICE +# +# Introspect port for debug +#introspect_port = 5995 + +[ANALYTICS_COLLECTOR] +# log file name and log_level +# log = /var/log/contrail/contrail-collector.log +# Log severity levels. Possible values are SYS_EMERG, SYS_ALERT, SYS_CRIT, +# SYS_ERR, SYS_WARN, SYS_NOTICE, SYS_INFO and SYS_DEBUG. Default is SYS_DEBUG +# log_level = SYS_NOTICE +# +# Introspect port for debug +# introspect_port = 8089 +# +# Listen address and port +# listen_port = 8086 +# listen_address = 0.0.0.0 + +[QUERY_ENGINE] +# log file name and log_level +# log = /var/log/contrail/contrail-query-engine.log +# Log severity levels. Possible values are SYS_EMERG, SYS_ALERT, SYS_CRIT, +# SYS_ERR, SYS_WARN, SYS_NOTICE, SYS_INFO and SYS_DEBUG. Default is SYS_DEBUG +# log_level = SYS_NOTICE +# +# Introspect port for debug +# introspect_port = 8091 + +[SNMP_COLLECTOR] +# log file name and log_level +# log = /var/log/contrail/contrail-snmp-collector.log +# Log severity levels. Possible values are SYS_EMERG, SYS_ALERT, SYS_CRIT, +# SYS_ERR, SYS_WARN, SYS_NOTICE, SYS_INFO and SYS_DEBUG. Default is SYS_DEBUG +# log_level = SYS_NOTICE +# +# Introspect port for debug +# introspect_port = 5920 +# +# scan_frequency=600 +# +# fast_scan_frequency=60 + +[TOPOLOGY] +# log file name and log_level +# log = /var/log/contrail/contrail-topology.log +# Log severity levels. Possible values are SYS_EMERG, SYS_ALERT, SYS_CRIT, +# SYS_ERR, SYS_WARN, SYS_NOTICE, SYS_INFO and SYS_DEBUG. Default is SYS_DEBUG +# log_level = SYS_NOTICE +# +# Introspect port for debug +# introspect_port = 5921 + +[RABBITMQ] +user = {{ rabbitmq_user }} +password = {{ rabbitmq_password }} +vhost = {{ rabbitmq_vhost }} |
