aboutsummaryrefslogtreecommitdiffstats
path: root/contrail-analytics/hooks/charmhelpers/fetch
diff options
context:
space:
mode:
Diffstat (limited to 'contrail-analytics/hooks/charmhelpers/fetch')
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/__init__.py205
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py165
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py76
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/centos.py171
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/giturl.py69
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/snap.py122
-rw-r--r--contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py568
7 files changed, 1376 insertions, 0 deletions
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/__init__.py b/contrail-analytics/hooks/charmhelpers/fetch/__init__.py
new file mode 100644
index 0000000..480a627
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/__init__.py
@@ -0,0 +1,205 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+from charmhelpers.osplatform import get_platform
+from yaml import safe_load
+from charmhelpers.core.hookenv import (
+ config,
+ log,
+)
+
+import six
+if six.PY3:
+ from urllib.parse import urlparse, urlunparse
+else:
+ from urlparse import urlparse, urlunparse
+
+
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+ 'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+ 'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+ 'charmhelpers.fetch.giturl.GitUrlFetchHandler',
+)
+
+
+class SourceConfigError(Exception):
+ pass
+
+
+class UnhandledSource(Exception):
+ pass
+
+
+class AptLockError(Exception):
+ pass
+
+
+class GPGKeyError(Exception):
+ """Exception occurs when a GPG key cannot be fetched or used. The message
+ indicates what the problem is.
+ """
+ pass
+
+
+class BaseFetchHandler(object):
+
+ """Base class for FetchHandler implementations in fetch plugins"""
+
+ def can_handle(self, source):
+ """Returns True if the source can be handled. Otherwise returns
+ a string explaining why it cannot"""
+ return "Wrong source type"
+
+ def install(self, source):
+ """Try to download and unpack the source. Return the path to the
+ unpacked files or raise UnhandledSource."""
+ raise UnhandledSource("Wrong source type {}".format(source))
+
+ def parse_url(self, url):
+ return urlparse(url)
+
+ def base_url(self, url):
+ """Return url without querystring or fragment"""
+ parts = list(self.parse_url(url))
+ parts[4:] = ['' for i in parts[4:]]
+ return urlunparse(parts)
+
+
+__platform__ = get_platform()
+module = "charmhelpers.fetch.%s" % __platform__
+fetch = importlib.import_module(module)
+
+filter_installed_packages = fetch.filter_installed_packages
+install = fetch.apt_install
+upgrade = fetch.apt_upgrade
+update = _fetch_update = fetch.apt_update
+purge = fetch.apt_purge
+add_source = fetch.add_source
+
+if __platform__ == "ubuntu":
+ apt_cache = fetch.apt_cache
+ apt_install = fetch.apt_install
+ apt_update = fetch.apt_update
+ apt_upgrade = fetch.apt_upgrade
+ apt_purge = fetch.apt_purge
+ apt_mark = fetch.apt_mark
+ apt_hold = fetch.apt_hold
+ apt_unhold = fetch.apt_unhold
+ import_key = fetch.import_key
+ get_upstream_version = fetch.get_upstream_version
+elif __platform__ == "centos":
+ yum_search = fetch.yum_search
+
+
+def configure_sources(update=False,
+ sources_var='install_sources',
+ keys_var='install_keys'):
+ """Configure multiple sources from charm configuration.
+
+ The lists are encoded as yaml fragments in the configuration.
+ The fragment needs to be included as a string. Sources and their
+ corresponding keys are of the types supported by add_source().
+
+ Example config:
+ install_sources: |
+ - "ppa:foo"
+ - "http://example.com/repo precise main"
+ install_keys: |
+ - null
+ - "a1b2c3d4"
+
+ Note that 'null' (a.k.a. None) should not be quoted.
+ """
+ sources = safe_load((config(sources_var) or '').strip()) or []
+ keys = safe_load((config(keys_var) or '').strip()) or None
+
+ if isinstance(sources, six.string_types):
+ sources = [sources]
+
+ if keys is None:
+ for source in sources:
+ add_source(source, None)
+ else:
+ if isinstance(keys, six.string_types):
+ keys = [keys]
+
+ if len(sources) != len(keys):
+ raise SourceConfigError(
+ 'Install sources and keys lists are different lengths')
+ for source, key in zip(sources, keys):
+ add_source(source, key)
+ if update:
+ _fetch_update(fatal=True)
+
+
+def install_remote(source, *args, **kwargs):
+ """Install a file tree from a remote source.
+
+ The specified source should be a url of the form:
+ scheme://[host]/path[#[option=value][&...]]
+
+ Schemes supported are based on this modules submodules.
+ Options supported are submodule-specific.
+ Additional arguments are passed through to the submodule.
+
+ For example::
+
+ dest = install_remote('http://example.com/archive.tgz',
+ checksum='deadbeef',
+ hash_type='sha1')
+
+ This will download `archive.tgz`, validate it using SHA1 and, if
+ the file is ok, extract it and return the directory in which it
+ was extracted. If the checksum fails, it will raise
+ :class:`charmhelpers.core.host.ChecksumError`.
+ """
+ # We ONLY check for True here because can_handle may return a string
+ # explaining why it can't handle a given source.
+ handlers = [h for h in plugins() if h.can_handle(source) is True]
+ for handler in handlers:
+ try:
+ return handler.install(source, *args, **kwargs)
+ except UnhandledSource as e:
+ log('Install source attempt unsuccessful: {}'.format(e),
+ level='WARNING')
+ raise UnhandledSource("No handler found for source {}".format(source))
+
+
+def install_from_config(config_var_name):
+ """Install a file from config."""
+ charm_config = config()
+ source = charm_config[config_var_name]
+ return install_remote(source)
+
+
+def plugins(fetch_handlers=None):
+ if not fetch_handlers:
+ fetch_handlers = FETCH_HANDLERS
+ plugin_list = []
+ for handler_name in fetch_handlers:
+ package, classname = handler_name.rsplit('.', 1)
+ try:
+ handler_class = getattr(
+ importlib.import_module(package),
+ classname)
+ plugin_list.append(handler_class())
+ except NotImplementedError:
+ # Skip missing plugins so that they can be ommitted from
+ # installation if desired
+ log("FetchHandler {} not found, skipping plugin".format(
+ handler_name))
+ return plugin_list
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py b/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py
new file mode 100644
index 0000000..dd24f9e
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/archiveurl.py
@@ -0,0 +1,165 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import hashlib
+import re
+
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource
+)
+from charmhelpers.payload.archive import (
+ get_archive_handler,
+ extract,
+)
+from charmhelpers.core.host import mkdir, check_hash
+
+import six
+if six.PY3:
+ from urllib.request import (
+ build_opener, install_opener, urlopen, urlretrieve,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ )
+ from urllib.parse import urlparse, urlunparse, parse_qs
+ from urllib.error import URLError
+else:
+ from urllib import urlretrieve
+ from urllib2 import (
+ build_opener, install_opener, urlopen,
+ HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+ URLError
+ )
+ from urlparse import urlparse, urlunparse, parse_qs
+
+
+def splituser(host):
+ '''urllib.splituser(), but six's support of this seems broken'''
+ _userprog = re.compile('^(.*)@(.*)$')
+ match = _userprog.match(host)
+ if match:
+ return match.group(1, 2)
+ return None, host
+
+
+def splitpasswd(user):
+ '''urllib.splitpasswd(), but six's support of this is missing'''
+ _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+ match = _passwdprog.match(user)
+ if match:
+ return match.group(1, 2)
+ return user, None
+
+
+class ArchiveUrlFetchHandler(BaseFetchHandler):
+ """
+ Handler to download archive files from arbitrary URLs.
+
+ Can fetch from http, https, ftp, and file URLs.
+
+ Can install either tarballs (.tar, .tgz, .tbz2, etc) or zip files.
+
+ Installs the contents of the archive in $CHARM_DIR/fetched/.
+ """
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+ # XXX: Why is this returning a boolean and a string? It's
+ # doomed to fail since "bool(can_handle('foo://'))" will be True.
+ return "Wrong source type"
+ if get_archive_handler(self.base_url(source)):
+ return True
+ return False
+
+ def download(self, source, dest):
+ """
+ Download an archive file.
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local path location to download archive file to.
+ """
+ # propogate all exceptions
+ # URLError, OSError, etc
+ proto, netloc, path, params, query, fragment = urlparse(source)
+ if proto in ('http', 'https'):
+ auth, barehost = splituser(netloc)
+ if auth is not None:
+ source = urlunparse((proto, barehost, path, params, query, fragment))
+ username, password = splitpasswd(auth)
+ passman = HTTPPasswordMgrWithDefaultRealm()
+ # Realm is set to None in add_password to force the username and password
+ # to be used whatever the realm
+ passman.add_password(None, source, username, password)
+ authhandler = HTTPBasicAuthHandler(passman)
+ opener = build_opener(authhandler)
+ install_opener(opener)
+ response = urlopen(source)
+ try:
+ with open(dest, 'wb') as dest_file:
+ dest_file.write(response.read())
+ except Exception as e:
+ if os.path.isfile(dest):
+ os.unlink(dest)
+ raise e
+
+ # Mandatory file validation via Sha1 or MD5 hashing.
+ def download_and_validate(self, url, hashsum, validate="sha1"):
+ tempfile, headers = urlretrieve(url)
+ check_hash(tempfile, hashsum, validate)
+ return tempfile
+
+ def install(self, source, dest=None, checksum=None, hash_type='sha1'):
+ """
+ Download and install an archive file, with optional checksum validation.
+
+ The checksum can also be given on the `source` URL's fragment.
+ For example::
+
+ handler.install('http://example.com/file.tgz#sha1=deadbeef')
+
+ :param str source: URL pointing to an archive file.
+ :param str dest: Local destination path to install to. If not given,
+ installs to `$CHARM_DIR/archives/archive_file_name`.
+ :param str checksum: If given, validate the archive file after download.
+ :param str hash_type: Algorithm used to generate `checksum`.
+ Can be any hash alrgorithm supported by :mod:`hashlib`,
+ such as md5, sha1, sha256, sha512, etc.
+
+ """
+ url_parts = self.parse_url(source)
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+ if not os.path.exists(dest_dir):
+ mkdir(dest_dir, perms=0o755)
+ dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
+ try:
+ self.download(source, dld_file)
+ except URLError as e:
+ raise UnhandledSource(e.reason)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ options = parse_qs(url_parts.fragment)
+ for key, value in options.items():
+ if not six.PY3:
+ algorithms = hashlib.algorithms
+ else:
+ algorithms = hashlib.algorithms_available
+ if key in algorithms:
+ if len(value) != 1:
+ raise TypeError(
+ "Expected 1 hash value, not %d" % len(value))
+ expected = value[0]
+ check_hash(dld_file, expected, key)
+ if checksum:
+ check_hash(dld_file, checksum, hash_type)
+ return extract(dld_file, dest)
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py b/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py
new file mode 100644
index 0000000..07cd029
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/bzrurl.py
@@ -0,0 +1,76 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from subprocess import check_call
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource,
+ filter_installed_packages,
+ install,
+)
+from charmhelpers.core.host import mkdir
+
+
+if filter_installed_packages(['bzr']) != []:
+ install(['bzr'])
+ if filter_installed_packages(['bzr']) != []:
+ raise NotImplementedError('Unable to install bzr')
+
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+ """Handler for bazaar branches via generic and lp URLs."""
+
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ if url_parts.scheme not in ('bzr+ssh', 'lp', ''):
+ return False
+ elif not url_parts.scheme:
+ return os.path.exists(os.path.join(source, '.bzr'))
+ else:
+ return True
+
+ def branch(self, source, dest, revno=None):
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+ cmd_opts = []
+ if revno:
+ cmd_opts += ['-r', str(revno)]
+ if os.path.exists(dest):
+ cmd = ['bzr', 'pull']
+ cmd += cmd_opts
+ cmd += ['--overwrite', '-d', dest, source]
+ else:
+ cmd = ['bzr', 'branch']
+ cmd += cmd_opts
+ cmd += [source, dest]
+ check_call(cmd)
+
+ def install(self, source, dest=None, revno=None):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
+
+ if dest and not os.path.exists(dest):
+ mkdir(dest, perms=0o755)
+
+ try:
+ self.branch(source, dest_dir, revno)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/centos.py b/contrail-analytics/hooks/charmhelpers/fetch/centos.py
new file mode 100644
index 0000000..a91dcff
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/centos.py
@@ -0,0 +1,171 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import subprocess
+import os
+import time
+import six
+import yum
+
+from tempfile import NamedTemporaryFile
+from charmhelpers.core.hookenv import log
+
+YUM_NO_LOCK = 1 # The return code for "couldn't acquire lock" in YUM.
+YUM_NO_LOCK_RETRY_DELAY = 10 # Wait 10 seconds between apt lock checks.
+YUM_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
+
+
+def filter_installed_packages(packages):
+ """Return a list of packages that require installation."""
+ yb = yum.YumBase()
+ package_list = yb.doPackageLists()
+ temp_cache = {p.base_package_name: 1 for p in package_list['installed']}
+
+ _pkgs = [p for p in packages if not temp_cache.get(p, False)]
+ return _pkgs
+
+
+def install(packages, options=None, fatal=False):
+ """Install one or more packages."""
+ cmd = ['yum', '--assumeyes']
+ if options is not None:
+ cmd.extend(options)
+ cmd.append('install')
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Installing {} with options: {}".format(packages,
+ options))
+ _run_yum_command(cmd, fatal)
+
+
+def upgrade(options=None, fatal=False, dist=False):
+ """Upgrade all packages."""
+ cmd = ['yum', '--assumeyes']
+ if options is not None:
+ cmd.extend(options)
+ cmd.append('upgrade')
+ log("Upgrading with options: {}".format(options))
+ _run_yum_command(cmd, fatal)
+
+
+def update(fatal=False):
+ """Update local yum cache."""
+ cmd = ['yum', '--assumeyes', 'update']
+ log("Update with fatal: {}".format(fatal))
+ _run_yum_command(cmd, fatal)
+
+
+def purge(packages, fatal=False):
+ """Purge one or more packages."""
+ cmd = ['yum', '--assumeyes', 'remove']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Purging {}".format(packages))
+ _run_yum_command(cmd, fatal)
+
+
+def yum_search(packages):
+ """Search for a package."""
+ output = {}
+ cmd = ['yum', 'search']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Searching for {}".format(packages))
+ result = subprocess.check_output(cmd)
+ for package in list(packages):
+ output[package] = package in result
+ return output
+
+
+def add_source(source, key=None):
+ """Add a package source to this system.
+
+ @param source: a URL with a rpm package
+
+ @param key: A key to be added to the system's keyring and used
+ to verify the signatures on packages. Ideally, this should be an
+ ASCII format GPG public key including the block headers. A GPG key
+ id may also be used, but be aware that only insecure protocols are
+ available to retrieve the actual public key from a public keyserver
+ placing your Juju environment at risk.
+ """
+ if source is None:
+ log('Source is not present. Skipping')
+ return
+
+ if source.startswith('http'):
+ directory = '/etc/yum.repos.d/'
+ for filename in os.listdir(directory):
+ with open(directory + filename, 'r') as rpm_file:
+ if source in rpm_file.read():
+ break
+ else:
+ log("Add source: {!r}".format(source))
+ # write in the charms.repo
+ with open(directory + 'Charms.repo', 'a') as rpm_file:
+ rpm_file.write('[%s]\n' % source[7:].replace('/', '_'))
+ rpm_file.write('name=%s\n' % source[7:])
+ rpm_file.write('baseurl=%s\n\n' % source)
+ else:
+ log("Unknown source: {!r}".format(source))
+
+ if key:
+ if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
+ with NamedTemporaryFile('w+') as key_file:
+ key_file.write(key)
+ key_file.flush()
+ key_file.seek(0)
+ subprocess.check_call(['rpm', '--import', key_file.name])
+ else:
+ subprocess.check_call(['rpm', '--import', key])
+
+
+def _run_yum_command(cmd, fatal=False):
+ """Run an YUM command.
+
+ Checks the output and retry if the fatal flag is set to True.
+
+ :param: cmd: str: The yum command to run.
+ :param: fatal: bool: Whether the command's output should be checked and
+ retried.
+ """
+ env = os.environ.copy()
+
+ if fatal:
+ retry_count = 0
+ result = None
+
+ # If the command is considered "fatal", we need to retry if the yum
+ # lock was not acquired.
+
+ while result is None or result == YUM_NO_LOCK:
+ try:
+ result = subprocess.check_call(cmd, env=env)
+ except subprocess.CalledProcessError as e:
+ retry_count = retry_count + 1
+ if retry_count > YUM_NO_LOCK_RETRY_COUNT:
+ raise
+ result = e.returncode
+ log("Couldn't acquire YUM lock. Will retry in {} seconds."
+ "".format(YUM_NO_LOCK_RETRY_DELAY))
+ time.sleep(YUM_NO_LOCK_RETRY_DELAY)
+
+ else:
+ subprocess.call(cmd, env=env)
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/giturl.py b/contrail-analytics/hooks/charmhelpers/fetch/giturl.py
new file mode 100644
index 0000000..4cf21bc
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/giturl.py
@@ -0,0 +1,69 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+from subprocess import check_call, CalledProcessError
+from charmhelpers.fetch import (
+ BaseFetchHandler,
+ UnhandledSource,
+ filter_installed_packages,
+ install,
+)
+
+if filter_installed_packages(['git']) != []:
+ install(['git'])
+ if filter_installed_packages(['git']) != []:
+ raise NotImplementedError('Unable to install git')
+
+
+class GitUrlFetchHandler(BaseFetchHandler):
+ """Handler for git branches via generic and github URLs."""
+
+ def can_handle(self, source):
+ url_parts = self.parse_url(source)
+ # TODO (mattyw) no support for ssh git@ yet
+ if url_parts.scheme not in ('http', 'https', 'git', ''):
+ return False
+ elif not url_parts.scheme:
+ return os.path.exists(os.path.join(source, '.git'))
+ else:
+ return True
+
+ def clone(self, source, dest, branch="master", depth=None):
+ if not self.can_handle(source):
+ raise UnhandledSource("Cannot handle {}".format(source))
+
+ if os.path.exists(dest):
+ cmd = ['git', '-C', dest, 'pull', source, branch]
+ else:
+ cmd = ['git', 'clone', source, dest, '--branch', branch]
+ if depth:
+ cmd.extend(['--depth', depth])
+ check_call(cmd)
+
+ def install(self, source, branch="master", dest=None, depth=None):
+ url_parts = self.parse_url(source)
+ branch_name = url_parts.path.strip("/").split("/")[-1]
+ if dest:
+ dest_dir = os.path.join(dest, branch_name)
+ else:
+ dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+ branch_name)
+ try:
+ self.clone(source, dest_dir, branch, depth)
+ except CalledProcessError as e:
+ raise UnhandledSource(e)
+ except OSError as e:
+ raise UnhandledSource(e.strerror)
+ return dest_dir
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/snap.py b/contrail-analytics/hooks/charmhelpers/fetch/snap.py
new file mode 100644
index 0000000..23c707b
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/snap.py
@@ -0,0 +1,122 @@
+# Copyright 2014-2017 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Charm helpers snap for classic charms.
+
+If writing reactive charms, use the snap layer:
+https://lists.ubuntu.com/archives/snapcraft/2016-September/001114.html
+"""
+import subprocess
+from os import environ
+from time import sleep
+from charmhelpers.core.hookenv import log
+
+__author__ = 'Joseph Borg <joseph.borg@canonical.com>'
+
+SNAP_NO_LOCK = 1 # The return code for "couldn't acquire lock" in Snap (hopefully this will be improved).
+SNAP_NO_LOCK_RETRY_DELAY = 10 # Wait X seconds between Snap lock checks.
+SNAP_NO_LOCK_RETRY_COUNT = 30 # Retry to acquire the lock X times.
+
+
+class CouldNotAcquireLockException(Exception):
+ pass
+
+
+def _snap_exec(commands):
+ """
+ Execute snap commands.
+
+ :param commands: List commands
+ :return: Integer exit code
+ """
+ assert type(commands) == list
+
+ retry_count = 0
+ return_code = None
+
+ while return_code is None or return_code == SNAP_NO_LOCK:
+ try:
+ return_code = subprocess.check_call(['snap'] + commands, env=environ)
+ except subprocess.CalledProcessError as e:
+ retry_count += + 1
+ if retry_count > SNAP_NO_LOCK_RETRY_COUNT:
+ raise CouldNotAcquireLockException('Could not aquire lock after %s attempts' % SNAP_NO_LOCK_RETRY_COUNT)
+ return_code = e.returncode
+ log('Snap failed to acquire lock, trying again in %s seconds.' % SNAP_NO_LOCK_RETRY_DELAY, level='WARN')
+ sleep(SNAP_NO_LOCK_RETRY_DELAY)
+
+ return return_code
+
+
+def snap_install(packages, *flags):
+ """
+ Install a snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to install command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Installing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with option(s) "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['install'] + flags + packages)
+
+
+def snap_remove(packages, *flags):
+ """
+ Remove a snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to remove command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Removing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with options "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['remove'] + flags + packages)
+
+
+def snap_refresh(packages, *flags):
+ """
+ Refresh / Update snap package.
+
+ :param packages: String or List String package name
+ :param flags: List String flags to pass to refresh command
+ :return: Integer return code from snap
+ """
+ if type(packages) is not list:
+ packages = [packages]
+
+ flags = list(flags)
+
+ message = 'Refreshing snap(s) "%s"' % ', '.join(packages)
+ if flags:
+ message += ' with options "%s"' % ', '.join(flags)
+
+ log(message, level='INFO')
+ return _snap_exec(['refresh'] + flags + packages)
diff --git a/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py b/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py
new file mode 100644
index 0000000..57b5fb6
--- /dev/null
+++ b/contrail-analytics/hooks/charmhelpers/fetch/ubuntu.py
@@ -0,0 +1,568 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections import OrderedDict
+import os
+import platform
+import re
+import six
+import time
+import subprocess
+from tempfile import NamedTemporaryFile
+
+from charmhelpers.core.host import (
+ lsb_release
+)
+from charmhelpers.core.hookenv import (
+ log,
+ DEBUG,
+)
+from charmhelpers.fetch import SourceConfigError, GPGKeyError
+
+PROPOSED_POCKET = (
+ "# Proposed\n"
+ "deb http://archive.ubuntu.com/ubuntu {}-proposed main universe "
+ "multiverse restricted\n")
+PROPOSED_PORTS_POCKET = (
+ "# Proposed\n"
+ "deb http://ports.ubuntu.com/ubuntu-ports {}-proposed main universe "
+ "multiverse restricted\n")
+# Only supports 64bit and ppc64 at the moment.
+ARCH_TO_PROPOSED_POCKET = {
+ 'x86_64': PROPOSED_POCKET,
+ 'ppc64le': PROPOSED_PORTS_POCKET,
+ 'aarch64': PROPOSED_PORTS_POCKET,
+}
+CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
+CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+CLOUD_ARCHIVE_POCKETS = {
+ # Folsom
+ 'folsom': 'precise-updates/folsom',
+ 'folsom/updates': 'precise-updates/folsom',
+ 'precise-folsom': 'precise-updates/folsom',
+ 'precise-folsom/updates': 'precise-updates/folsom',
+ 'precise-updates/folsom': 'precise-updates/folsom',
+ 'folsom/proposed': 'precise-proposed/folsom',
+ 'precise-folsom/proposed': 'precise-proposed/folsom',
+ 'precise-proposed/folsom': 'precise-proposed/folsom',
+ # Grizzly
+ 'grizzly': 'precise-updates/grizzly',
+ 'grizzly/updates': 'precise-updates/grizzly',
+ 'precise-grizzly': 'precise-updates/grizzly',
+ 'precise-grizzly/updates': 'precise-updates/grizzly',
+ 'precise-updates/grizzly': 'precise-updates/grizzly',
+ 'grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-grizzly/proposed': 'precise-proposed/grizzly',
+ 'precise-proposed/grizzly': 'precise-proposed/grizzly',
+ # Havana
+ 'havana': 'precise-updates/havana',
+ 'havana/updates': 'precise-updates/havana',
+ 'precise-havana': 'precise-updates/havana',
+ 'precise-havana/updates': 'precise-updates/havana',
+ 'precise-updates/havana': 'precise-updates/havana',
+ 'havana/proposed': 'precise-proposed/havana',
+ 'precise-havana/proposed': 'precise-proposed/havana',
+ 'precise-proposed/havana': 'precise-proposed/havana',
+ # Icehouse
+ 'icehouse': 'precise-updates/icehouse',
+ 'icehouse/updates': 'precise-updates/icehouse',
+ 'precise-icehouse': 'precise-updates/icehouse',
+ 'precise-icehouse/updates': 'precise-updates/icehouse',
+ 'precise-updates/icehouse': 'precise-updates/icehouse',
+ 'icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-icehouse/proposed': 'precise-proposed/icehouse',
+ 'precise-proposed/icehouse': 'precise-proposed/icehouse',
+ # Juno
+ 'juno': 'trusty-updates/juno',
+ 'juno/updates': 'trusty-updates/juno',
+ 'trusty-juno': 'trusty-updates/juno',
+ 'trusty-juno/updates': 'trusty-updates/juno',
+ 'trusty-updates/juno': 'trusty-updates/juno',
+ 'juno/proposed': 'trusty-proposed/juno',
+ 'trusty-juno/proposed': 'trusty-proposed/juno',
+ 'trusty-proposed/juno': 'trusty-proposed/juno',
+ # Kilo
+ 'kilo': 'trusty-updates/kilo',
+ 'kilo/updates': 'trusty-updates/kilo',
+ 'trusty-kilo': 'trusty-updates/kilo',
+ 'trusty-kilo/updates': 'trusty-updates/kilo',
+ 'trusty-updates/kilo': 'trusty-updates/kilo',
+ 'kilo/proposed': 'trusty-proposed/kilo',
+ 'trusty-kilo/proposed': 'trusty-proposed/kilo',
+ 'trusty-proposed/kilo': 'trusty-proposed/kilo',
+ # Liberty
+ 'liberty': 'trusty-updates/liberty',
+ 'liberty/updates': 'trusty-updates/liberty',
+ 'trusty-liberty': 'trusty-updates/liberty',
+ 'trusty-liberty/updates': 'trusty-updates/liberty',
+ 'trusty-updates/liberty': 'trusty-updates/liberty',
+ 'liberty/proposed': 'trusty-proposed/liberty',
+ 'trusty-liberty/proposed': 'trusty-proposed/liberty',
+ 'trusty-proposed/liberty': 'trusty-proposed/liberty',
+ # Mitaka
+ 'mitaka': 'trusty-updates/mitaka',
+ 'mitaka/updates': 'trusty-updates/mitaka',
+ 'trusty-mitaka': 'trusty-updates/mitaka',
+ 'trusty-mitaka/updates': 'trusty-updates/mitaka',
+ 'trusty-updates/mitaka': 'trusty-updates/mitaka',
+ 'mitaka/proposed': 'trusty-proposed/mitaka',
+ 'trusty-mitaka/proposed': 'trusty-proposed/mitaka',
+ 'trusty-proposed/mitaka': 'trusty-proposed/mitaka',
+ # Newton
+ 'newton': 'xenial-updates/newton',
+ 'newton/updates': 'xenial-updates/newton',
+ 'xenial-newton': 'xenial-updates/newton',
+ 'xenial-newton/updates': 'xenial-updates/newton',
+ 'xenial-updates/newton': 'xenial-updates/newton',
+ 'newton/proposed': 'xenial-proposed/newton',
+ 'xenial-newton/proposed': 'xenial-proposed/newton',
+ 'xenial-proposed/newton': 'xenial-proposed/newton',
+ # Ocata
+ 'ocata': 'xenial-updates/ocata',
+ 'ocata/updates': 'xenial-updates/ocata',
+ 'xenial-ocata': 'xenial-updates/ocata',
+ 'xenial-ocata/updates': 'xenial-updates/ocata',
+ 'xenial-updates/ocata': 'xenial-updates/ocata',
+ 'ocata/proposed': 'xenial-proposed/ocata',
+ 'xenial-ocata/proposed': 'xenial-proposed/ocata',
+ 'xenial-ocata/newton': 'xenial-proposed/ocata',
+ # Pike
+ 'pike': 'xenial-updates/pike',
+ 'xenial-pike': 'xenial-updates/pike',
+ 'xenial-pike/updates': 'xenial-updates/pike',
+ 'xenial-updates/pike': 'xenial-updates/pike',
+ 'pike/proposed': 'xenial-proposed/pike',
+ 'xenial-pike/proposed': 'xenial-proposed/pike',
+ 'xenial-pike/newton': 'xenial-proposed/pike',
+ # Queens
+ 'queens': 'xenial-updates/queens',
+ 'xenial-queens': 'xenial-updates/queens',
+ 'xenial-queens/updates': 'xenial-updates/queens',
+ 'xenial-updates/queens': 'xenial-updates/queens',
+ 'queens/proposed': 'xenial-proposed/queens',
+ 'xenial-queens/proposed': 'xenial-proposed/queens',
+ 'xenial-queens/newton': 'xenial-proposed/queens',
+}
+
+
+APT_NO_LOCK = 100 # The return code for "couldn't acquire lock" in APT.
+CMD_RETRY_DELAY = 10 # Wait 10 seconds between command retries.
+CMD_RETRY_COUNT = 30 # Retry a failing fatal command X times.
+
+
+def filter_installed_packages(packages):
+ """Return a list of packages that require installation."""
+ cache = apt_cache()
+ _pkgs = []
+ for package in packages:
+ try:
+ p = cache[package]
+ p.current_ver or _pkgs.append(package)
+ except KeyError:
+ log('Package {} has no installation candidate.'.format(package),
+ level='WARNING')
+ _pkgs.append(package)
+ return _pkgs
+
+
+def apt_cache(in_memory=True, progress=None):
+ """Build and return an apt cache."""
+ from apt import apt_pkg
+ apt_pkg.init()
+ if in_memory:
+ apt_pkg.config.set("Dir::Cache::pkgcache", "")
+ apt_pkg.config.set("Dir::Cache::srcpkgcache", "")
+ return apt_pkg.Cache(progress)
+
+
+def apt_install(packages, options=None, fatal=False):
+ """Install one or more packages."""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ cmd.append('install')
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Installing {} with options: {}".format(packages,
+ options))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_upgrade(options=None, fatal=False, dist=False):
+ """Upgrade all packages."""
+ if options is None:
+ options = ['--option=Dpkg::Options::=--force-confold']
+
+ cmd = ['apt-get', '--assume-yes']
+ cmd.extend(options)
+ if dist:
+ cmd.append('dist-upgrade')
+ else:
+ cmd.append('upgrade')
+ log("Upgrading with options: {}".format(options))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_update(fatal=False):
+ """Update local apt cache."""
+ cmd = ['apt-get', 'update']
+ _run_apt_command(cmd, fatal)
+
+
+def apt_purge(packages, fatal=False):
+ """Purge one or more packages."""
+ cmd = ['apt-get', '--assume-yes', 'purge']
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+ log("Purging {}".format(packages))
+ _run_apt_command(cmd, fatal)
+
+
+def apt_mark(packages, mark, fatal=False):
+ """Flag one or more packages using apt-mark."""
+ log("Marking {} as {}".format(packages, mark))
+ cmd = ['apt-mark', mark]
+ if isinstance(packages, six.string_types):
+ cmd.append(packages)
+ else:
+ cmd.extend(packages)
+
+ if fatal:
+ subprocess.check_call(cmd, universal_newlines=True)
+ else:
+ subprocess.call(cmd, universal_newlines=True)
+
+
+def apt_hold(packages, fatal=False):
+ return apt_mark(packages, 'hold', fatal=fatal)
+
+
+def apt_unhold(packages, fatal=False):
+ return apt_mark(packages, 'unhold', fatal=fatal)
+
+
+def import_key(keyid):
+ """Import a key in either ASCII Armor or Radix64 format.
+
+ `keyid` is either the keyid to fetch from a PGP server, or
+ the key in ASCII armor foramt.
+
+ :param keyid: String of key (or key id).
+ :raises: GPGKeyError if the key could not be imported
+ """
+ key = keyid.strip()
+ if (key.startswith('-----BEGIN PGP PUBLIC KEY BLOCK-----') and
+ key.endswith('-----END PGP PUBLIC KEY BLOCK-----')):
+ log("PGP key found (looks like ASCII Armor format)", level=DEBUG)
+ log("Importing ASCII Armor PGP key", level=DEBUG)
+ with NamedTemporaryFile() as keyfile:
+ with open(keyfile.name, 'w') as fd:
+ fd.write(key)
+ fd.write("\n")
+ cmd = ['apt-key', 'add', keyfile.name]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ error = "Error importing PGP key '{}'".format(key)
+ log(error)
+ raise GPGKeyError(error)
+ else:
+ log("PGP key found (looks like Radix64 format)", level=DEBUG)
+ log("Importing PGP key from keyserver", level=DEBUG)
+ cmd = ['apt-key', 'adv', '--keyserver',
+ 'hkp://keyserver.ubuntu.com:80', '--recv-keys', key]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ error = "Error importing PGP key '{}'".format(key)
+ log(error)
+ raise GPGKeyError(error)
+
+
+def add_source(source, key=None, fail_invalid=False):
+ """Add a package source to this system.
+
+ @param source: a URL or sources.list entry, as supported by
+ add-apt-repository(1). Examples::
+
+ ppa:charmers/example
+ deb https://stub:key@private.example.com/ubuntu trusty main
+
+ In addition:
+ 'proposed:' may be used to enable the standard 'proposed'
+ pocket for the release.
+ 'cloud:' may be used to activate official cloud archive pockets,
+ such as 'cloud:icehouse'
+ 'distro' may be used as a noop
+
+ Full list of source specifications supported by the function are:
+
+ 'distro': A NOP; i.e. it has no effect.
+ 'proposed': the proposed deb spec [2] is wrtten to
+ /etc/apt/sources.list/proposed
+ 'distro-proposed': adds <version>-proposed to the debs [2]
+ 'ppa:<ppa-name>': add-apt-repository --yes <ppa_name>
+ 'deb <deb-spec>': add-apt-repository --yes deb <deb-spec>
+ 'http://....': add-apt-repository --yes http://...
+ 'cloud-archive:<spec>': add-apt-repository -yes cloud-archive:<spec>
+ 'cloud:<release>[-staging]': specify a Cloud Archive pocket <release> with
+ optional staging version. If staging is used then the staging PPA [2]
+ with be used. If staging is NOT used then the cloud archive [3] will be
+ added, and the 'ubuntu-cloud-keyring' package will be added for the
+ current distro.
+
+ Otherwise the source is not recognised and this is logged to the juju log.
+ However, no error is raised, unless sys_error_on_exit is True.
+
+ [1] deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+ where {} is replaced with the derived pocket name.
+ [2] deb http://archive.ubuntu.com/ubuntu {}-proposed \
+ main universe multiverse restricted
+ where {} is replaced with the lsb_release codename (e.g. xenial)
+ [3] deb http://ubuntu-cloud.archive.canonical.com/ubuntu <pocket>
+ to /etc/apt/sources.list.d/cloud-archive-list
+
+ @param key: A key to be added to the system's APT keyring and used
+ to verify the signatures on packages. Ideally, this should be an
+ ASCII format GPG public key including the block headers. A GPG key
+ id may also be used, but be aware that only insecure protocols are
+ available to retrieve the actual public key from a public keyserver
+ placing your Juju environment at risk. ppa and cloud archive keys
+ are securely added automtically, so sould not be provided.
+
+ @param fail_invalid: (boolean) if True, then the function raises a
+ SourceConfigError is there is no matching installation source.
+
+ @raises SourceConfigError() if for cloud:<pocket>, the <pocket> is not a
+ valid pocket in CLOUD_ARCHIVE_POCKETS
+ """
+ _mapping = OrderedDict([
+ (r"^distro$", lambda: None), # This is a NOP
+ (r"^(?:proposed|distro-proposed)$", _add_proposed),
+ (r"^cloud-archive:(.*)$", _add_apt_repository),
+ (r"^((?:deb |http:|https:|ppa:).*)$", _add_apt_repository),
+ (r"^cloud:(.*)-(.*)\/staging$", _add_cloud_staging),
+ (r"^cloud:(.*)-(.*)$", _add_cloud_distro_check),
+ (r"^cloud:(.*)$", _add_cloud_pocket),
+ ])
+ if source is None:
+ source = ''
+ for r, fn in six.iteritems(_mapping):
+ m = re.match(r, source)
+ if m:
+ # call the assoicated function with the captured groups
+ # raises SourceConfigError on error.
+ fn(*m.groups())
+ if key:
+ try:
+ import_key(key)
+ except GPGKeyError as e:
+ raise SourceConfigError(str(e))
+ break
+ else:
+ # nothing matched. log an error and maybe sys.exit
+ err = "Unknown source: {!r}".format(source)
+ log(err)
+ if fail_invalid:
+ raise SourceConfigError(err)
+
+
+def _add_proposed():
+ """Add the PROPOSED_POCKET as /etc/apt/source.list.d/proposed.list
+
+ Uses lsb_release()['DISTRIB_CODENAME'] to determine the correct staza for
+ the deb line.
+
+ For intel architecutres PROPOSED_POCKET is used for the release, but for
+ other architectures PROPOSED_PORTS_POCKET is used for the release.
+ """
+ release = lsb_release()['DISTRIB_CODENAME']
+ arch = platform.machine()
+ if arch not in six.iterkeys(ARCH_TO_PROPOSED_POCKET):
+ raise SourceConfigError("Arch {} not supported for (distro-)proposed"
+ .format(arch))
+ with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+ apt.write(ARCH_TO_PROPOSED_POCKET[arch].format(release))
+
+
+def _add_apt_repository(spec):
+ """Add the spec using add_apt_repository
+
+ :param spec: the parameter to pass to add_apt_repository
+ """
+ _run_with_retries(['add-apt-repository', '--yes', spec])
+
+
+def _add_cloud_pocket(pocket):
+ """Add a cloud pocket as /etc/apt/sources.d/cloud-archive.list
+
+ Note that this overwrites the existing file if there is one.
+
+ This function also converts the simple pocket in to the actual pocket using
+ the CLOUD_ARCHIVE_POCKETS mapping.
+
+ :param pocket: string representing the pocket to add a deb spec for.
+ :raises: SourceConfigError if the cloud pocket doesn't exist or the
+ requested release doesn't match the current distro version.
+ """
+ apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
+ fatal=True)
+ if pocket not in CLOUD_ARCHIVE_POCKETS:
+ raise SourceConfigError(
+ 'Unsupported cloud: source option %s' %
+ pocket)
+ actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+ with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+ apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+
+
+def _add_cloud_staging(cloud_archive_release, openstack_release):
+ """Add the cloud staging repository which is in
+ ppa:ubuntu-cloud-archive/<openstack_release>-staging
+
+ This function checks that the cloud_archive_release matches the current
+ codename for the distro that charm is being installed on.
+
+ :param cloud_archive_release: string, codename for the release.
+ :param openstack_release: String, codename for the openstack release.
+ :raises: SourceConfigError if the cloud_archive_release doesn't match the
+ current version of the os.
+ """
+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
+ ppa = 'ppa:ubuntu-cloud-archive/{}-staging'.format(openstack_release)
+ cmd = 'add-apt-repository -y {}'.format(ppa)
+ _run_with_retries(cmd.split(' '))
+
+
+def _add_cloud_distro_check(cloud_archive_release, openstack_release):
+ """Add the cloud pocket, but also check the cloud_archive_release against
+ the current distro, and use the openstack_release as the full lookup.
+
+ This just calls _add_cloud_pocket() with the openstack_release as pocket
+ to get the correct cloud-archive.list for dpkg to work with.
+
+ :param cloud_archive_release:String, codename for the distro release.
+ :param openstack_release: String, spec for the release to look up in the
+ CLOUD_ARCHIVE_POCKETS
+ :raises: SourceConfigError if this is the wrong distro, or the pocket spec
+ doesn't exist.
+ """
+ _verify_is_ubuntu_rel(cloud_archive_release, openstack_release)
+ _add_cloud_pocket("{}-{}".format(cloud_archive_release, openstack_release))
+
+
+def _verify_is_ubuntu_rel(release, os_release):
+ """Verify that the release is in the same as the current ubuntu release.
+
+ :param release: String, lowercase for the release.
+ :param os_release: String, the os_release being asked for
+ :raises: SourceConfigError if the release is not the same as the ubuntu
+ release.
+ """
+ ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+ if release != ubuntu_rel:
+ raise SourceConfigError(
+ 'Invalid Cloud Archive release specified: {}-{} on this Ubuntu'
+ 'version ({})'.format(release, os_release, ubuntu_rel))
+
+
+def _run_with_retries(cmd, max_retries=CMD_RETRY_COUNT, retry_exitcodes=(1,),
+ retry_message="", cmd_env=None):
+ """Run a command and retry until success or max_retries is reached.
+
+ :param: cmd: str: The apt command to run.
+ :param: max_retries: int: The number of retries to attempt on a fatal
+ command. Defaults to CMD_RETRY_COUNT.
+ :param: retry_exitcodes: tuple: Optional additional exit codes to retry.
+ Defaults to retry on exit code 1.
+ :param: retry_message: str: Optional log prefix emitted during retries.
+ :param: cmd_env: dict: Environment variables to add to the command run.
+ """
+
+ env = None
+ kwargs = {}
+ if cmd_env:
+ env = os.environ.copy()
+ env.update(cmd_env)
+ kwargs['env'] = env
+
+ if not retry_message:
+ retry_message = "Failed executing '{}'".format(" ".join(cmd))
+ retry_message += ". Will retry in {} seconds".format(CMD_RETRY_DELAY)
+
+ retry_count = 0
+ result = None
+
+ retry_results = (None,) + retry_exitcodes
+ while result in retry_results:
+ try:
+ # result = subprocess.check_call(cmd, env=env)
+ result = subprocess.check_call(cmd, **kwargs)
+ except subprocess.CalledProcessError as e:
+ retry_count = retry_count + 1
+ if retry_count > max_retries:
+ raise
+ result = e.returncode
+ log(retry_message)
+ time.sleep(CMD_RETRY_DELAY)
+
+
+def _run_apt_command(cmd, fatal=False):
+ """Run an apt command with optional retries.
+
+ :param: cmd: str: The apt command to run.
+ :param: fatal: bool: Whether the command's output should be checked and
+ retried.
+ """
+ # Provide DEBIAN_FRONTEND=noninteractive if not present in the environment.
+ cmd_env = {
+ 'DEBIAN_FRONTEND': os.environ.get('DEBIAN_FRONTEND', 'noninteractive')}
+
+ if fatal:
+ _run_with_retries(
+ cmd, cmd_env=cmd_env, retry_exitcodes=(1, APT_NO_LOCK,),
+ retry_message="Couldn't acquire DPKG lock")
+ else:
+ env = os.environ.copy()
+ env.update(cmd_env)
+ subprocess.call(cmd, env=env)
+
+
+def get_upstream_version(package):
+ """Determine upstream version based on installed package
+
+ @returns None (if not installed) or the upstream version
+ """
+ import apt_pkg
+ cache = apt_cache()
+ try:
+ pkg = cache[package]
+ except:
+ # the package is unknown to the current apt cache.
+ return None
+
+ if not pkg.current_ver:
+ # package is known, but no version is currently installed.
+ return None
+
+ return apt_pkg.upstream_version(pkg.current_ver.ver_str)