1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
|
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others.
# mskalski@mirantis.com
# Alexandru.Avadanii@enea.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
"""Build multiarch partial local Ubuntu mirror using packetary"""
##############################################################################
# Design quirks / workarounds:
# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from
# the local mirror; which only uses the "main" component from the first
# repository, i.e. does not include "updates"/"security".
# In order to fullfill all debootstrap dependencies in "main" repo, we will
# do an extra packetary run using a reduced scope:
# - only "main" component of the first mirror;
# - reduced package dependency list (without MOS/OPNFV plugin deps).
# 2. If repo structure is not mandatory to be in sync with official mirrors,
# we can mitigate the issue by "merging" all repo-components into a single
# "main".
##############################################################################
# Mirror build steps (for EACH architecture in UBUNTU_ARCH):
# 1. Collect bootstrap package deps from fuel-agent's <settings.yaml.sample>;
# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>;
# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror);
# 4. Inherit enviroment variable(s) for mirror URLs, paths etc.
# - Allow arch-specific overrides for each env var;
# 5. Mirror config is defined based on common config + OPNFV overrides;
# - Convert old configuration format to packetary style where needed;
# 6. Package lists are defined based on common config + OPNFV deps;
# - Keep track of "main" packages separately, required by debootstrap;
# 7. Clone/update all mirror components;
# 8. IF mirror merging is disabled:
# - Clone/update "main" mirror component (fix missing debootstrap deps);
# 9. IF mirror merging is enabled:
# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs;
# - Run `packetary create` on the set of downloaded packages, merging
# them on the fly into a single-component mirror;
##############################################################################
from copy import deepcopy
import os
import shutil
import sys
import yaml
from contextlib import contextmanager
from cStringIO import StringIO
from packetary.cli.app import main
@contextmanager
def capture_stdout(output):
"""Context manager for capturing stdout"""
stdout = sys.stdout
sys.stdout = output
yield
sys.stdout = stdout
# FIXME: Find a better approach for eliminating duplicate logs than this
def force_logger_reload():
"""Force logger reload (ugly hack to prevent log duplication)"""
for mod in sys.modules.keys():
if mod.startswith('logging'):
try:
reload(sys.modules[mod])
except:
pass
def get_unres_pkgs(architecture, cfg_mirror):
"""Determine missing package dependecies for a mirror defition"""
unresolved_pkgs = list()
packetary_output = StringIO()
with capture_stdout(packetary_output):
main('unresolved -a {0} -r {1} -c name version --sep ;'
.format(_ARCH[architecture], cfg_mirror).split(' '))
for dep_pkg in packetary_output.getvalue().splitlines():
if dep_pkg.startswith('#'):
continue
dep = dep_pkg.split(';')
unresolved_pkgs += [{'name': dep[0], 'version': dep[1]}]
force_logger_reload()
return unresolved_pkgs
def from_legacy_pkglist(legacy_pkglist):
"""Package list conversion from `old fuel-mirror` to `packetary` style"""
pkglist = list()
for pkg in legacy_pkglist:
pkglist += [{'name': pkg}]
return pkglist
def to_legacy_pkglist(pkglist):
"""Package list conversion from `packetary` style to `old fuel-mirror`"""
legacy_pkglist = list()
for pkg in pkglist:
legacy_pkglist.append(pkg['name'])
return legacy_pkglist
def legacy_diff(base_pkglist, new_pkglist, requester, architecture):
"""Package list diff (old format)"""
diff_set = set(new_pkglist)
if base_pkglist:
diff_set -= set(base_pkglist)
if diff_set:
print(' * {0} requires new packages for architecture [{1}]: {2}'
.format(requester, architecture, ', '.join(diff_set)))
return list(diff_set)
def do_local_repo(architecture, cfg_repo, cfg_packages_paths):
"""Create single-component local repo (one architecture per call)"""
# Packetary does not use a global config file, so pass old settings here.
main('create -t deb -a {0} --repository {1} --package-files {2}'
' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
.format(_ARCH[architecture], cfg_repo, cfg_packages_paths).split(' '))
force_logger_reload()
def do_partial_mirror(architecture, cfg_mirror, cfg_packages):
"""Clone partial local mirror (one architecture per call)"""
# Note: '-d .' is ignored, as each mirror defines its own path.
main('clone -t deb -a {0} -r {1} -R {2} -d .'
' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
.format(_ARCH[architecture], cfg_mirror, cfg_packages).split(' '))
force_logger_reload()
def write_cfg_file(cfg_mirror, data):
"""Write configuration (yaml) file (package list / mirror defition)"""
with open(cfg_mirror, 'w') as outfile:
outfile.write(yaml.safe_dump(data, default_flow_style=False))
def get_env(env_var, architecture=None):
"""Evaluate architecture-specific overrides of env vars"""
if architecture:
env_var_arch = '{0}_{1}'.format(env_var, architecture)
if os.environ.get(env_var_arch):
return os.environ[env_var_arch]
if os.environ.get(env_var):
return os.environ[env_var]
return None
# Architecture name mapping (dpkg:packetary) for packetary CLI invocation
_ARCH = {
"i386": "i386",
"amd64": "x86_64",
"arm64": "aarch64",
}
# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages)
CFG_D = 'opnfv_config'
CFG_OPNFV = 'opnfv_config.yaml'
MOS_VERSION = get_env('MOS_VERSION')
UBUNTU_ARCH = get_env('UBUNTU_ARCH')
MIRROR_UBUNTU_PATH = get_env('MIRROR_UBUNTU_OPNFV_PATH')
MIRROR_UBUNTU_TMP_PATH = '{0}.tmp'.format(MIRROR_UBUNTU_PATH)
MIRROR_UBUNTU_MERGE = get_env('MIRROR_UBUNTU_MERGE')
CFG_MM_UBUNTU = '{0}/ubuntu_mirror_local.yaml'.format(CFG_D)
FUEL_BOOTSTRAP_CLI_FILE = open('fuel-agent/contrib/fuel_bootstrap/'
'fuel_bootstrap_cli/fuel_bootstrap/settings.yaml.sample').read()
FUEL_BOOTSTRAP_CLI = yaml.load(FUEL_BOOTSTRAP_CLI_FILE)
FIXTURE_FILE = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read()
FIXTURE = yaml.load(FIXTURE_FILE)
OPNFV_CFG_YAML = open(CFG_OPNFV).read()
OPNFV_CFG = yaml.load(OPNFV_CFG_YAML)
# Create local partial mirror using packetary, one arch at a time
for arch in UBUNTU_ARCH.split(' '):
# Mirror / Package env vars, arch-overrideable
mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch)
mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch)
mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch)
plugins = get_env('BUILD_FUEL_PLUGINS', arch)
if plugins is None:
plugins = get_env('PLUGINS', arch)
# Mirror / Package list configuration files (arch-specific)
cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(CFG_D, arch)
cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(CFG_D, arch)
cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(CFG_D, arch)
cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(CFG_D, arch)
cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(CFG_D, arch)
# Mirror config fork before customizing (arch-specific)
arch_mos = 'mos_{0}'.format(arch)
arch_ubuntu = 'ubuntu_{0}'.format(arch)
arch_packages = 'packages_{0}'.format(arch)
OPNFV_CFG['groups'][arch_mos] = deepcopy(OPNFV_CFG['groups']['mos'])
OPNFV_CFG['groups'][arch_ubuntu] = deepcopy(OPNFV_CFG['groups']['ubuntu'])
OPNFV_CFG[arch_packages] = OPNFV_CFG['packages']
# Mirror config update & conversion to packetary input
group_main_ubuntu = dict()
for group in OPNFV_CFG['groups'][arch_mos]:
group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
group['suite'] = group['suite'].replace('$mos_version', MOS_VERSION)
for group in OPNFV_CFG['groups'][arch_ubuntu]:
group['uri'] = mirror_ubuntu
# FIXME: At `create`, packetary insists on copying all pkgs to dest dir,
# so configure it for another dir, which will replace the orig.
group['path'] = MIRROR_UBUNTU_TMP_PATH
if not group_main_ubuntu and 'main' in group:
group_main_ubuntu = [deepcopy(group)]
group_main_ubuntu[0]['section'] = ['main']
# Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main]
write_cfg_file(cfg_m_mos, OPNFV_CFG['groups'][arch_mos])
write_cfg_file(cfg_m_ubuntu, OPNFV_CFG['groups'][arch_ubuntu])
if MIRROR_UBUNTU_MERGE is None:
write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu)
else:
# FIXME: For multiarch, only one dump would be enough
group_main_ubuntu[0]['origin'] = 'Ubuntu'
group_main_ubuntu[0]['path'] = MIRROR_UBUNTU_PATH
group_main_ubuntu[0]['uri'] = MIRROR_UBUNTU_PATH
write_cfg_file(CFG_MM_UBUNTU, group_main_ubuntu[0])
# Collect package dependencies from:
## 1. fuel_bootstrap_cli (bootstrap image additional packages)
legacy_unresolved = legacy_diff(None, FUEL_BOOTSTRAP_CLI['packages'] + [
FUEL_BOOTSTRAP_CLI['kernel_flavor'],
FUEL_BOOTSTRAP_CLI['kernel_flavor'].replace('image', 'headers')],
'Bootstrap', arch)
## 2. openstack.yaml FIXTURE definition (default target image packages)
for release in FIXTURE:
editable = release['fields']['attributes_metadata']['editable']
if 'provision' in editable and 'packages' in editable['provision']:
release_pkgs = editable['provision']['packages']['value'].split()
legacy_unresolved += legacy_diff(legacy_unresolved, release_pkgs,
'Release {0}'.format(release['fields']['name']), arch)
## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs)
unresolved = dict()
unresolved['mandatory'] = 'exact'
unresolved['packages'] = from_legacy_pkglist(legacy_unresolved)
if 'packages' in OPNFV_CFG:
legacy_diff(legacy_unresolved, to_legacy_pkglist(OPNFV_CFG['packages']),
'OPNFV config', arch)
unresolved['packages'] += OPNFV_CFG['packages']
# OPNFV plugins dependency resolution
if plugins:
for plugin in plugins.split():
path = "../{}/packages.yaml".format(plugin)
if os.path.isfile(path):
f = open(path).read()
plugin_yaml = yaml.load(f)
new_pkgs = legacy_diff(
to_legacy_pkglist(unresolved['packages']),
plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch)
unresolved['packages'] += from_legacy_pkglist(new_pkgs)
# Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps)
if MIRROR_UBUNTU_MERGE is None:
write_cfg_file(cfg_p_ubuntu_main, unresolved)
# Mirror package list (full, including MOS/OPNFV plugin deps)
unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos)
write_cfg_file(cfg_p_ubuntu, unresolved)
do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu)
if MIRROR_UBUNTU_MERGE is None:
# Ubuntu[main] must be evaluated after Ubuntu
do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main)
if MIRROR_UBUNTU_MERGE is None:
shutil.move(MIRROR_UBUNTU_TMP_PATH, MIRROR_UBUNTU_PATH)
else:
# Construct single-component mirror from all components
for arch in UBUNTU_ARCH.split(' '):
cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(CFG_D, arch)
# OPNFV blacklist
opnfv_blacklist = to_legacy_pkglist(OPNFV_CFG['opnfv_blacklist'])
# FIXME: We need scanpackages to omit older DEBs
# Inspired from http://askubuntu.com/questions/198474/
os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | '
'grep -e "^Filename:" | sed "s|Filename: |- file://|g" | '
'grep -v -E "\/({2})_" > {3}'
.format(arch, MIRROR_UBUNTU_TMP_PATH,
'|'.join(opnfv_blacklist), cfg_pp_ubuntu))
do_local_repo(arch, CFG_MM_UBUNTU, cfg_pp_ubuntu)
shutil.rmtree(MIRROR_UBUNTU_TMP_PATH)
|
