1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
|
# Copyright 2015 Canonical Ltd.
#
# This file is part of the Cassandra Charm for Juju.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranties of
# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR
# PURPOSE. See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from functools import wraps
import json
import os
import shutil
import subprocess
import tempfile
import time
import amulet
import yaml
class AmuletFixture(amulet.Deployment):
def __init__(self, series, verbose=False):
if verbose:
super(AmuletFixture, self).__init__(series=series)
else:
# We use a wrapper around juju-deployer so we can fix how it is
# invoked. In particular, turn off all the noise so we can
# actually read our test output.
juju_deployer = os.path.abspath(os.path.join(
os.path.dirname(__file__), os.pardir, 'lib',
'juju-deployer-wrapper.py'))
super(AmuletFixture, self).__init__(series=series,
juju_deployer=juju_deployer)
assert self.series == series
def setUp(self):
self._temp_dirs = []
self.reset_environment(force=True)
# Repackage our charm to a temporary directory, allowing us
# to strip our virtualenv symlinks that would otherwise cause
# juju to abort. We also strip the .bzr directory, working
# around Bug #1394078.
self.repackage_charm()
# Fix amulet.Deployment so it doesn't depend on environment
# variables or the current working directory, but rather the
# environment we have introspected.
with open(os.path.join(self.charm_dir, 'metadata.yaml'), 'r') as s:
self.charm_name = yaml.safe_load(s)['name']
self.charm_cache.test_charm = None
self.charm_cache.fetch(self.charm_name, self.charm_dir,
series=self.series)
# Explicitly reset $JUJU_REPOSITORY to ensure amulet and
# juju-deployer does not mess with the real one, per Bug #1393792
self.org_repo = os.environ.get('JUJU_REPOSITORY', None)
temp_repo = tempfile.mkdtemp(suffix='.repo')
self._temp_dirs.append(temp_repo)
os.environ['JUJU_REPOSITORY'] = temp_repo
os.makedirs(os.path.join(temp_repo, self.series), mode=0o700)
def tearDown(self, reset_environment=True):
if reset_environment:
self.reset_environment()
if self.org_repo is None:
del os.environ['JUJU_REPOSITORY']
else:
os.environ['JUJU_REPOSITORY'] = self.org_repo
def deploy(self, timeout=None):
'''Deploying or updating the configured system.
Invokes amulet.Deployer.setup with a nicer name and standard
timeout handling.
'''
if timeout is None:
timeout = int(os.environ.get('AMULET_TIMEOUT', 900))
# juju-deployer is buried under here, and has race conditions.
# Sleep a bit before invoking it, so its cached view of the
# environment matches reality.
time.sleep(15)
# If setUp fails, tearDown is never called leaving the
# environment setup. This is useful for debugging.
self.setup(timeout=timeout)
self.wait(timeout=timeout)
def __del__(self):
for temp_dir in self._temp_dirs:
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir, ignore_errors=True)
def get_status(self):
try:
raw = subprocess.check_output(['juju', 'status', '--format=json'],
universal_newlines=True)
except subprocess.CalledProcessError as x:
print(x.output)
raise
if raw:
return json.loads(raw)
return None
def wait(self, timeout=None):
'''Wait until the environment has reached a stable state.'''
if timeout is None:
timeout = int(os.environ.get('AMULET_TIMEOUT', 900))
cmd = ['timeout', str(timeout), 'juju', 'wait', '-q']
try:
subprocess.check_output(cmd, universal_newlines=True)
except subprocess.CalledProcessError as x:
print(x.output)
raise
def reset_environment(self, force=False):
if force:
status = self.get_status()
machines = [m for m in status.get('machines', {}).keys()
if m != '0']
if machines:
subprocess.call(['juju', 'destroy-machine',
'--force'] + machines,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
fails = dict()
while True:
status = self.get_status()
service_items = status.get('services', {}).items()
if not service_items:
break
for service_name, service in service_items:
if service.get('life', '') not in ('dying', 'dead'):
subprocess.call(['juju', 'destroy-service', service_name],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for unit_name, unit in service.get('units', {}).items():
if unit.get('agent-state', None) == 'error':
if force:
# If any units have failed hooks, unstick them.
# This should no longer happen now we are
# using the 'destroy-machine --force' command
# earlier.
try:
subprocess.check_output(
['juju', 'resolved', unit_name],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
# A previous 'resolved' call make cause a
# subsequent one to fail if it is still
# being processed. However, we need to keep
# retrying because after a successful
# resolution a subsequent hook may cause an
# error state.
pass
else:
fails[unit_name] = unit
time.sleep(1)
harvest_machines = []
for machine, state in status.get('machines', {}).items():
if machine != "0" and state.get('life') not in ('dying', 'dead'):
harvest_machines.append(machine)
if harvest_machines:
cmd = ['juju', 'remove-machine', '--force'] + harvest_machines
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
if fails:
raise Exception("Teardown failed", fails)
def repackage_charm(self):
"""Mirror the charm into a staging area.
We do this to work around issues with Amulet, juju-deployer
and juju. In particular:
- symlinks in the Python virtual env pointing outside of the
charm directory.
- odd bzr interactions, such as tests being run on the committed
version of the charm, rather than the working tree.
Returns the test charm directory.
"""
# Find the charm_dir we are testing
src_charm_dir = os.path.dirname(__file__)
while True:
if os.path.exists(os.path.join(src_charm_dir,
'metadata.yaml')):
break
assert src_charm_dir != os.sep, 'metadata.yaml not found'
src_charm_dir = os.path.abspath(os.path.join(src_charm_dir,
os.pardir))
with open(os.path.join(src_charm_dir, 'metadata.yaml'), 'r') as s:
self.charm_name = yaml.safe_load(s)['name']
repack_root = tempfile.mkdtemp(suffix='.charm')
self._temp_dirs.append(repack_root)
# juju-deployer now requires the series in the path when
# deploying from an absolute path.
repack_root = os.path.join(repack_root, self.series)
os.makedirs(repack_root, mode=0o700)
self.charm_dir = os.path.join(repack_root, self.charm_name)
# Ignore .bzr to work around weird bzr interactions with
# juju-deployer, per Bug #1394078, and ignore .venv
# due to a) it containing symlinks juju will reject and b) to avoid
# infinite recursion.
shutil.copytree(src_charm_dir, self.charm_dir, symlinks=True,
ignore=shutil.ignore_patterns('.venv?', '.bzr'))
# Bug #1417097 means we need to monkey patch Amulet for now.
real_juju = amulet.helpers.juju
@wraps(real_juju)
def patched_juju(args, env=None):
args = [str(a) for a in args]
return real_juju(args, env)
amulet.helpers.juju = patched_juju
amulet.deployer.juju = patched_juju
|
