aboutsummaryrefslogtreecommitdiffstats
path: root/deploy/deploy.py
blob: f491929e08189dea7e612586a8e234e52eaa13af (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
#!/usr/bin/python
###############################################################################
# Copyright (c) 2015 Ericsson AB and others.
# szilard.cserey@ericsson.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
###############################################################################


import os
import io
import re
import sys
import yaml
import errno
import signal
import netaddr

from dea import DeploymentEnvironmentAdapter
from dha import DeploymentHardwareAdapter
from install_fuel_master import InstallFuelMaster
from deploy_env import CloudDeploy
from execution_environment import ExecutionEnvironment

from common import (
    log,
    exec_cmd,
    err,
    warn,
    check_file_exists,
    check_dir_exists,
    create_dir_if_not_exists,
    delete,
    check_if_root,
    ArgParser,
)

FUEL_VM = 'fuel'
PATCH_DIR = 'fuel_patch'
WORK_DIR = '~/deploy'
CWD = os.getcwd()
MOUNT_STATE_VAR = 'AUTODEPLOY_ISO_MOUNTED'


class cd:

    def __init__(self, new_path):
        self.new_path = os.path.expanduser(new_path)

    def __enter__(self):
        self.saved_path = CWD
        os.chdir(self.new_path)

    def __exit__(self, etype, value, traceback):
        os.chdir(self.saved_path)


class AutoDeploy(object):

    def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
                 cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
                 dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
        self.no_fuel = no_fuel
        self.fuel_only = fuel_only
        self.no_health_check = no_health_check
        self.cleanup_only = cleanup_only
        self.cleanup = cleanup
        self.storage_dir = storage_dir
        self.pxe_bridge = pxe_bridge
        self.iso_file = iso_file
        self.dea_file = dea_file
        self.dha_file = dha_file
        self.fuel_plugins_dir = fuel_plugins_dir
        self.fuel_plugins_conf_dir = fuel_plugins_conf_dir
        self.no_plugins = no_plugins
        self.deploy_timeout = deploy_timeout
        self.no_deploy_environment = no_deploy_environment
        self.deploy_log = deploy_log
        self.dea = (DeploymentEnvironmentAdapter(dea_file)
                    if not cleanup_only else None)
        self.dha = DeploymentHardwareAdapter(dha_file)
        self.fuel_conf = {}
        self.fuel_node_id = self.dha.get_fuel_node_id()
        self.fuel_username, self.fuel_password = self.dha.get_fuel_access()
        self.tmp_dir = None

    def modify_ip(self, ip_addr, index, val):
        ip_str = str(netaddr.IPAddress(ip_addr))
        decimal_list = map(int, ip_str.split('.'))
        decimal_list[index] = val
        return '.'.join(map(str, decimal_list))

    def collect_fuel_info(self):
        self.fuel_conf['ip'] = self.dea.get_fuel_ip()
        self.fuel_conf['gw'] = self.dea.get_fuel_gateway()
        self.fuel_conf['dns1'] = self.dea.get_fuel_dns()
        self.fuel_conf['netmask'] = self.dea.get_fuel_netmask()
        self.fuel_conf['hostname'] = self.dea.get_fuel_hostname()
        self.fuel_conf['showmenu'] = 'yes'

    def install_fuel_master(self):
        log('Install Fuel Master')
        new_iso = ('%s/deploy-%s'
                   % (self.tmp_dir, os.path.basename(self.iso_file)))
        self.patch_iso(new_iso)
        self.iso_file = new_iso
        self.install_iso()

    def delete_old_fuel_env(self):
        log('Delete old Fuel Master environments if present')
        try:
            old_dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
                                  self.fuel_username, self.fuel_password,
                                  self.dea_file, self.fuel_plugins_conf_dir,
                                  WORK_DIR, self.no_health_check,
                                  self.deploy_timeout,
                                  self.no_deploy_environment, self.deploy_log)
            with old_dep.ssh:
                old_dep.check_previous_installation()
        except Exception as e:
            log('Could not delete old env: %s' % str(e))

    def install_iso(self):
        fuel = InstallFuelMaster(self.dea_file, self.dha_file,
                                 self.fuel_conf['ip'], self.fuel_username,
                                 self.fuel_password, self.fuel_node_id,
                                 self.iso_file, WORK_DIR,
                                 self.fuel_plugins_dir, self.no_plugins)
        fuel.install()

    def patch_iso(self, new_iso):
        tmp_orig_dir = '%s/origiso' % self.tmp_dir
        tmp_new_dir = '%s/newiso' % self.tmp_dir
        try:
            self.copy(tmp_orig_dir, tmp_new_dir)
            self.patch(tmp_new_dir, new_iso)
        except Exception as e:
            exec_cmd('fusermount -u %s' % tmp_orig_dir, False)
            os.environ.pop(MOUNT_STATE_VAR, None)
            delete(self.tmp_dir)
            err(e)

    def copy(self, tmp_orig_dir, tmp_new_dir):
        log('Copying...')
        os.makedirs(tmp_orig_dir)
        os.makedirs(tmp_new_dir)
        exec_cmd('fuseiso %s %s' % (self.iso_file, tmp_orig_dir))
        os.environ[MOUNT_STATE_VAR] = tmp_orig_dir
        with cd(tmp_orig_dir):
            exec_cmd('find . | cpio -pd %s' % tmp_new_dir)
        exec_cmd('fusermount -u %s' % tmp_orig_dir)
        os.environ.pop(MOUNT_STATE_VAR, None)
        delete(tmp_orig_dir)
        exec_cmd('chmod -R 755 %s' % tmp_new_dir)

    def patch(self, tmp_new_dir, new_iso):
        log('Patching...')
        patch_dir = '%s/%s' % (CWD, PATCH_DIR)
        ks_path = '%s/ks.cfg.patch' % patch_dir

        with cd(tmp_new_dir):
            exec_cmd('cat %s | patch -p0' % ks_path)
            delete('.rr_moved')
            isolinux = 'isolinux/isolinux.cfg'
            log('isolinux.cfg before: %s'
                % exec_cmd('grep ip= %s' % isolinux))
            self.update_fuel_isolinux(isolinux)
            log('isolinux.cfg after: %s'
                % exec_cmd('grep ip= %s' % isolinux))

            iso_label = self.parse_iso_volume_label(self.iso_file)
            log('Volume label: %s' % iso_label)

            iso_linux_bin = 'isolinux/isolinux.bin'
            exec_cmd('mkisofs -quiet -r -J -R -b %s '
                     '-no-emul-boot -boot-load-size 4 '
                     '-boot-info-table -hide-rr-moved '
                     '-joliet-long '
                     '-x "lost+found:" -V %s -o %s .'
                     % (iso_linux_bin, iso_label, new_iso))

        delete(tmp_new_dir)

    def update_fuel_isolinux(self, file):
        with io.open(file) as f:
            data = f.read()
        for key, val in self.fuel_conf.iteritems():
            # skip replacing these keys, as the format is different
            if key in ['ip', 'gw', 'netmask', 'hostname']:
                continue

            pattern = r'%s=[^ ]\S+' % key
            replace = '%s=%s' % (key, val)
            data = re.sub(pattern, replace, data)

        # process networking parameters
        ip = ':'.join([self.fuel_conf['ip'],
                      '',
                      self.fuel_conf['gw'],
                      self.fuel_conf['netmask'],
                      self.fuel_conf['hostname'],
                      'eth0:off:::'])

        data = re.sub(r'ip=[^ ]\S+', 'ip=%s' % ip, data)

        with io.open(file, 'w') as f:
            f.write(data)

    def parse_iso_volume_label(self, iso_filename):
        label_line = exec_cmd('isoinfo -d -i %s | grep -i "Volume id: "' % iso_filename)
        # cut leading text: 'Volume id: '
        return label_line[11:]

    def deploy_env(self):
        dep = CloudDeploy(self.dea, self.dha, self.fuel_conf['ip'],
                          self.fuel_username, self.fuel_password,
                          self.dea_file, self.fuel_plugins_conf_dir,
                          WORK_DIR, self.no_health_check, self.deploy_timeout,
                          self.no_deploy_environment, self.deploy_log)
        return dep.deploy()

    def setup_execution_environment(self):
        exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
                                        self.dha_file, self.dea)
        exec_env.setup_environment()

    def cleanup_execution_environment(self):
        exec_env = ExecutionEnvironment(self.storage_dir, self.pxe_bridge,
                                        self.dha_file, self.dea)
        exec_env.cleanup_environment()

    def create_tmp_dir(self):
        self.tmp_dir = '%s/fueltmp' % CWD
        delete(self.tmp_dir)
        create_dir_if_not_exists(self.tmp_dir)

    def deploy(self):
        self.collect_fuel_info()
        if not self.no_fuel:
            self.delete_old_fuel_env()
            self.setup_execution_environment()
            self.create_tmp_dir()
            self.install_fuel_master()
        if not self.fuel_only:
            return self.deploy_env()
        # Exit status
        return 0

    def run(self):
        check_if_root()
        if self.cleanup_only:
            self.cleanup_execution_environment()
        else:
            deploy_success = self.deploy()
            if self.cleanup:
                self.cleanup_execution_environment()
            return deploy_success
        # Exit status
        return 0


def check_bridge(pxe_bridge, dha_path):
    # Assume that bridges on remote nodes exists, we could ssh but
    # the remote user might not have a login shell.
    if os.environ.get('LIBVIRT_DEFAULT_URI'):
        return

    with io.open(dha_path) as yaml_file:
        dha_struct = yaml.load(yaml_file)
    if dha_struct['adapter'] != 'libvirt':
        log('Using Linux Bridge %s for booting up the Fuel Master VM'
            % pxe_bridge)
        r = exec_cmd('ip link show %s' % pxe_bridge)
        if pxe_bridge in r and 'state DOWN' in r:
            err('Linux Bridge {0} is not Active, bring'
                ' it UP first: [ip link set dev {0} up]'.format(pxe_bridge))


def check_fuel_plugins_dir(dir):
    msg = None
    if not dir:
        msg = 'Fuel Plugins Directory not specified!'
    elif not os.path.isdir(dir):
        msg = 'Fuel Plugins Directory does not exist!'
    elif not os.listdir(dir):
        msg = 'Fuel Plugins Directory is empty!'
    if msg:
        warn('%s No external plugins will be installed!' % msg)


def parse_arguments():
    parser = ArgParser(prog='python %s' % __file__)
    parser.add_argument('-nf', dest='no_fuel', action='store_true',
                        default=False,
                        help='Do not install Fuel Master (and Node VMs when '
                             'using libvirt)')
    parser.add_argument('-nh', dest='no_health_check', action='store_true',
                        default=False,
                        help='Don\'t run health check after deployment')
    parser.add_argument('-fo', dest='fuel_only', action='store_true',
                        default=False,
                        help='Install Fuel Master only (and Node VMs when '
                             'using libvirt)')
    parser.add_argument('-co', dest='cleanup_only', action='store_true',
                        default=False,
                        help='Cleanup VMs and Virtual Networks according to '
                             'what is defined in DHA')
    parser.add_argument('-c', dest='cleanup', action='store_true',
                        default=False,
                        help='Cleanup after deploy')
    if {'-iso', '-dea', '-dha', '-h'}.intersection(sys.argv):
        parser.add_argument('-iso', dest='iso_file', action='store', nargs='?',
                            default='%s/OPNFV.iso' % CWD,
                            help='ISO File [default: OPNFV.iso]')
        parser.add_argument('-dea', dest='dea_file', action='store', nargs='?',
                            default='%s/dea.yaml' % CWD,
                            help='Deployment Environment Adapter: dea.yaml')
        parser.add_argument('-dha', dest='dha_file', action='store', nargs='?',
                            default='%s/dha.yaml' % CWD,
                            help='Deployment Hardware Adapter: dha.yaml')
    else:
        parser.add_argument('iso_file', action='store', nargs='?',
                            default='%s/OPNFV.iso' % CWD,
                            help='ISO File [default: OPNFV.iso]')
        parser.add_argument('dea_file', action='store', nargs='?',
                            default='%s/dea.yaml' % CWD,
                            help='Deployment Environment Adapter: dea.yaml')
        parser.add_argument('dha_file', action='store', nargs='?',
                            default='%s/dha.yaml' % CWD,
                            help='Deployment Hardware Adapter: dha.yaml')
    parser.add_argument('-s', dest='storage_dir', action='store',
                        default='%s/images' % CWD,
                        help='Storage Directory [default: images]')
    parser.add_argument('-b', dest='pxe_bridge', action='append',
                        default=[],
                        help='Linux Bridge for booting up the Fuel Master VM '
                             '[default: pxebr]')
    parser.add_argument('-p', dest='fuel_plugins_dir', action='store',
                        help='Fuel Plugins directory')
    parser.add_argument('-pc', dest='fuel_plugins_conf_dir', action='store',
                        help='Fuel Plugins Configuration directory')
    parser.add_argument('-np', dest='no_plugins', action='store_true',
                        default=False, help='Do not install Fuel Plugins')
    parser.add_argument('-dt', dest='deploy_timeout', action='store',
                        default=240, help='Deployment timeout (in minutes) '
                        '[default: 240]')
    parser.add_argument('-nde', dest='no_deploy_environment',
                        action='store_true', default=False,
                        help=('Do not launch environment deployment'))
    parser.add_argument('-log', dest='deploy_log',
                        action='store', default='../ci/.',
                        help=('Path and name of the deployment log archive'))

    args = parser.parse_args()
    log(args)

    if not args.pxe_bridge:
        args.pxe_bridge = ['pxebr']

    check_file_exists(args.dha_file)

    check_dir_exists(os.path.dirname(args.deploy_log))

    if not args.cleanup_only:
        check_file_exists(args.dea_file)
        check_fuel_plugins_dir(args.fuel_plugins_dir)

    iso_abs_path = os.path.abspath(args.iso_file)
    if not args.no_fuel and not args.cleanup_only:
        log('Using OPNFV ISO file: %s' % iso_abs_path)
        check_file_exists(iso_abs_path)
        log('Using image directory: %s' % args.storage_dir)
        create_dir_if_not_exists(args.storage_dir)
        for bridge in args.pxe_bridge:
            check_bridge(bridge, args.dha_file)


    kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
              'no_health_check': args.no_health_check,
              'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
              'storage_dir': args.storage_dir, 'pxe_bridge': args.pxe_bridge,
              'iso_file': iso_abs_path, 'dea_file': args.dea_file,
              'dha_file': args.dha_file,
              'fuel_plugins_dir': args.fuel_plugins_dir,
              'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
              'no_plugins': args.no_plugins,
              'deploy_timeout': args.deploy_timeout,
              'no_deploy_environment': args.no_deploy_environment,
              'deploy_log': args.deploy_log}
    return kwargs


def handle_signals(signal_num, frame):
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    log('Caught signal %s, cleaning up and exiting.' % signal_num)

    mount_point = os.environ.get(MOUNT_STATE_VAR)
    if mount_point:
        log('Unmounting ISO from "%s"' % mount_point)
        # Prevent 'Device or resource busy' errors when unmounting
        os.chdir('/')
        exec_cmd('fusermount -u %s' % mount_point, True)
        # Be nice and remove our environment variable, even though the OS would
        # would clean it up anyway
        os.environ.pop(MOUNT_STATE_VAR)

    sys.exit(1)


def main():
    signal.signal(signal.SIGINT, handle_signals)
    signal.signal(signal.SIGTERM, handle_signals)
    kwargs = parse_arguments()
    d = AutoDeploy(**kwargs)
    sys.exit(d.run())

if __name__ == '__main__':
    main()