aboutsummaryrefslogtreecommitdiffstats
path: root/patches/opnfv-fuel/0015-Add-pre-install-purge-support-for-base-image.patch
blob: 28a31396574647ed0b671d5fb4eaeff48d649b3b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Date: Fri, 24 Nov 2017 20:28:01 +0100
Subject: [PATCH] Add pre-{install,purge} support for base image

Extend <lib.sh> and its invocation from <ci.deploy.sh> with
support for modifying foundation node VMs base image prior to
using it with:
- additional APT GPG keys;
- additional APT repos;
- packages to pre-install;
- packages to pre-remove;
- (non-configurable) cloud init datasource via NoCloud only,
  so VCP VMs won't wait for metadata service;

While at it, re-use the resulting image as a base for another
round of pre-patching (same operations as above are supported)
to provide a base image for VCP VMs.

Add AArch64-specific configuration based on new mechanisms:
- pre-install linux-image-generic-hwe-16.04-edge (and headers)
  for foundation node and VCP (common) image (also requires new
  repo and its key);
- pre-install cloud-init for VCP image (it should already be
  installed, but script needs non-empty config for VCP to create
  the VCP image and transfer it over to Salt Master);

NOTE: cloud-init is required on VCP VMs for DHCP on 1st iface.

JIRA: FUEL-309

Change-Id: I7dcaf0ffd9c57009133c6d339496ec831ab14375
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---
 .gitignore                                         |   1 +
 ci/deploy.sh                                       |  23 +++-
 mcp/config/scenario/defaults-aarch64.yaml          |  18 +++
 .../cluster/all-mcp-ocata-common/aarch64/init.yml  |   2 +-
 mcp/scripts/lib.sh                                 | 137 ++++++++++++++++++++-
 mcp/scripts/salt.sh                                |  12 ++
 6 files changed, 185 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4e90f32..38b4440 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,4 @@
 **/mcp/scripts/mcp.rsa*
 **/mcp/scripts/user-data.sh
 **/mcp/scripts/net_mcpcontrol.xml
+**/mcp/scripts/*.img
diff --git a/ci/deploy.sh b/ci/deploy.sh
index b13e18e..427850f 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -15,6 +15,7 @@
 do_exit () {
     local RC=$?
     clean
+    cleanup_mounts
     if [ ${RC} -eq 0 ]; then
         notify "\n[OK] MCP: Openstack installation finished succesfully!\n\n" 2
     else
@@ -158,6 +159,7 @@ NO_DEPLOY_ENVIRONMENT=${NO_DEPLOY_ENVIRONMENT:-0}
 ERASE_ENV=${ERASE_ENV:-0}

 source "${DEPLOY_DIR}/globals.sh"
+source "${DEPLOY_DIR}/lib.sh"

 #
 # END of variables to customize
@@ -340,7 +342,6 @@ fi

 # Get required infra deployment data
 set +x
-source lib.sh
 eval "$(parse_yaml "${SCENARIO_DIR}/defaults-$(uname -i).yaml")"
 eval "$(parse_yaml "${SCENARIO_DIR}/${DEPLOY_TYPE}/${DEPLOY_SCENARIO}.yaml")"
 eval "$(parse_yaml "${LOCAL_PDF_RECLASS}")"
@@ -358,6 +359,23 @@ for node in "${virtual_nodes[@]}"; do
 done
 virtual_nodes_data=${virtual_nodes_data%|}

+# Serialize repos, packages to (pre-)install/remove for:
+# - foundation node VM base image (virtual: all VMs, baremetal: cfg01|mas01)
+# - virtualized control plane VM base image (only when VCP is used)
+base_image_flavors=common
+if [[ "${cluster_states[*]}" =~ virtual_control ]]; then
+  base_image_flavors+=" control"
+fi
+for sc in ${base_image_flavors}; do
+  for va in apt_keys apt_repos pkg_install pkg_remove; do
+    key=virtual_${sc}_${va}
+    eval "${key}=\${${key}[@]// /|}"
+    eval "${key}=\${${key}// /,}"
+    virtual_repos_pkgs+="${!key}^"
+  done
+done
+virtual_repos_pkgs=${virtual_repos_pkgs%^}
+
 # Expand reclass and virsh network templates
 for tp in "${RECLASS_CLUSTER_DIR}/all-mcp-ocata-common/opnfv/"*.template \
     net_*.template; do
@@ -413,7 +431,8 @@ elif [ ${USE_EXISTING_INFRA} -gt 0 ]; then
     check_connection
 else
     generate_ssh_key
-    prepare_vms "${base_image}" "${STORAGE_DIR}" "${virtual_nodes[@]}"
+    prepare_vms "${base_image}" "${STORAGE_DIR}" "${virtual_repos_pkgs}" \
+      "${virtual_nodes[@]}"
     create_networks "${OPNFV_BRIDGES[@]}"
     create_vms "${STORAGE_DIR}" "${virtual_nodes_data}" "${OPNFV_BRIDGES[@]}"
     update_mcpcontrol_network
diff --git a/mcp/config/scenario/defaults-aarch64.yaml b/mcp/config/scenario/defaults-aarch64.yaml
index 24a4037..d989819 100644
--- a/mcp/config/scenario/defaults-aarch64.yaml
+++ b/mcp/config/scenario/defaults-aarch64.yaml
@@ -11,3 +11,21 @@ virtual:
   default:
     vcpus: 6
     ram: 4096
+  common:
+    apt:
+      keys:
+        - https://linux.enea.com/mcp-repos/ocata/xenial/archive-mcpocata.key
+      repos:
+        # <repo name> <repo prio> deb [arch=<arch>] <repo url> <repo dist> <repo comp>
+        - armband_openstack 1100 deb [arch=arm64] http://linux.enea.com/mcp-repos/ocata/xenial ocata main
+        - armband_mk_openstack 1100 deb [arch=arm64] http://linux.enea.com/apt-mk/xenial nightly ocata
+        # NOTE(armband): Empty repo, keep commented out as reference
+        # - armband_mcp_extra 1100 deb [arch=arm64] http://linux.enea.com/apt-mk/xenial nightly extra
+    pkg:
+      install:
+        - linux-image-generic-hwe-16.04-edge
+        - linux-headers-generic-hwe-16.04-edge
+  control:
+    pkg:
+      install:
+        - cloud-init
diff --git a/mcp/reclass/classes/cluster/all-mcp-ocata-common/aarch64/init.yml b/mcp/reclass/classes/cluster/all-mcp-ocata-common/aarch64/init.yml
index b5b78ec..0350afa 100644
--- a/mcp/reclass/classes/cluster/all-mcp-ocata-common/aarch64/init.yml
+++ b/mcp/reclass/classes/cluster/all-mcp-ocata-common/aarch64/init.yml
@@ -8,7 +8,7 @@
 ---
 parameters:
   _param:
-    salt_control_xenial_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img
+    salt_control_xenial_image: salt://salt/files/control/images/base_image_opnfv_fuel_vcp.img

     # VMs spawned on Foundation Node / Jump Host net ifaces (max 4)
     opnfv_fn_vm_primary_interface: enp1s0
diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index c6d5d26..654f45e 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -1,4 +1,5 @@
 #!/bin/bash -e
+# shellcheck disable=SC2155,SC1001
 ##############################################################################
 # Copyright (c) 2017 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -11,7 +12,6 @@
 #

 function generate_ssh_key {
-  # shellcheck disable=SC2155
   local mcp_ssh_key=$(basename "${SSH_KEY}")
   local user=${USER}
   if [ -n "${SUDO_USER}" ] && [ "${SUDO_USER}" != 'root' ]; then
@@ -35,6 +35,110 @@ function get_base_image {
   wget -P "${image_dir}" -N "${base_image}"
 }

+function mount_image {
+  local image=$1
+  local image_dir=$2
+  OPNFV_MNT_DIR="${image_dir}/ubuntu"
+
+  sudo modprobe nbd loop
+  # Find free nbd, loop devices
+  for dev in '/sys/class/block/nbd'*; do
+    if [ "$(cat "${dev}/size")" = '0' ]; then
+      OPNFV_NBD_DEV=/dev/$(basename "${dev}")
+      break
+    fi
+  done
+  OPNFV_LOOP_DEV=$(losetup -f)
+  export OPNFV_MNT_DIR OPNFV_LOOP_DEV
+  [ -n "${OPNFV_NBD_DEV}" ] && [ -n "${OPNFV_LOOP_DEV}" ] || exit 1
+  sudo qemu-nbd --connect="${OPNFV_NBD_DEV}" --aio=native --cache=none \
+    "${image_dir}/${image}"
+  sleep 5 # /dev/nbdNp1 takes some time to come up
+  # grub-update does not like /dev/nbd*, so use a loop device to work around it
+  # Hardcode partition index to 1, unlikely to change for Ubuntu UCA image
+  sudo losetup "${OPNFV_LOOP_DEV}" "${OPNFV_NBD_DEV}p1"
+  mkdir -p "${OPNFV_MNT_DIR}"
+  sudo mount "${OPNFV_LOOP_DEV}" "${OPNFV_MNT_DIR}"
+  sudo mount -t proc proc "${OPNFV_MNT_DIR}/proc"
+  sudo mount -t sysfs sys "${OPNFV_MNT_DIR}/sys"
+  sudo mount -o bind /dev "${OPNFV_MNT_DIR}/dev"
+  sudo mkdir -p "${OPNFV_MNT_DIR}/run/resolvconf"
+  sudo cp /etc/resolv.conf "${OPNFV_MNT_DIR}/run/resolvconf"
+  echo "GRUB_DISABLE_OS_PROBER=true" | \
+    sudo tee -a "${OPNFV_MNT_DIR}/etc/default/grub"
+}
+
+function apt_repos_pkgs_image {
+  local apt_key_urls=(${1//,/ })
+  local all_repos=(${2//,/ })
+  local pkgs_i=(${3//,/ })
+  local pkgs_r=(${4//,/ })
+  [ -n "${OPNFV_MNT_DIR}" ] || exit 1
+
+  # APT keys
+  if [ "${#apt_key_urls[@]}" -gt 0 ]; then
+    for apt_key in "${apt_key_urls[@]}"; do
+      sudo chroot "${OPNFV_MNT_DIR}" /bin/bash -c \
+        "wget -qO - '${apt_key}' | apt-key add -"
+    done
+  fi
+  # Additional repositories
+  for repo_line in "${all_repos[@]}"; do
+    # <repo_name>|<repo prio>|deb|[arch=<arch>]|<repo url>|<dist>|<repo comp>
+    local repo=(${repo_line//|/ })
+    [ "${#repo[@]}" -gt 5 ] || continue
+    # NOTE: Names and formatting are compatible with Salt linux.system.repo
+    cat <<-EOF | sudo tee "${OPNFV_MNT_DIR}/etc/apt/preferences.d/${repo[0]}"
+
+		Package: *
+		Pin: release a=${repo[-2]}
+		Pin-Priority: ${repo[1]}
+
+		EOF
+    echo "${repo[@]:2}" | sudo tee \
+      "${OPNFV_MNT_DIR}/etc/apt/sources.list.d/${repo[0]}.list"
+  done
+  # Install packages
+  if [ "${#pkgs_i[@]}" -gt 0 ]; then
+    sudo DEBIAN_FRONTEND="noninteractive" \
+      chroot "${OPNFV_MNT_DIR}" apt-get update
+    sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \
+      chroot "${OPNFV_MNT_DIR}" apt-get install -y "${pkgs_i[@]}"
+  fi
+  # Remove packages
+  if [ "${#pkgs_r[@]}" -gt 0 ]; then
+    sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \
+      chroot "${OPNFV_MNT_DIR}" apt-get purge -y "${pkgs_r[@]}"
+  fi
+  # Disable cloud-init metadata service datasource
+  sudo mkdir -p "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d"
+  echo "datasource_list: [ NoCloud, None ]" | sudo tee \
+    "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d/95_real_datasources.cfg"
+}
+
+function cleanup_mounts {
+  # Remove any mounts, loop and/or nbd devs created while patching base image
+  if [ -n "${OPNFV_MNT_DIR}" ] && [ -d "${OPNFV_MNT_DIR}" ]; then
+    if [ -f "${OPNFV_MNT_DIR}/boot/grub/grub.cfg" ]; then
+      # Grub thinks it's running from a live CD
+      sudo sed -i -e 's/^\s*set root=.*$//g' -e 's/^\s*loopback.*$//g' \
+        "${OPNFV_MNT_DIR}/boot/grub/grub.cfg"
+    fi
+    sudo rm -f "${OPNFV_MNT_DIR}/run/resolvconf/resolv.conf"
+    sync
+    if mountpoint -q "${OPNFV_MNT_DIR}"; then
+      sudo umount -l "${OPNFV_MNT_DIR}" || true
+    fi
+  fi
+  if [ -n "${OPNFV_LOOP_DEV}" ] && \
+    losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then
+      sudo losetup -d "${OPNFV_LOOP_DEV}"
+  fi
+  if [ -n "${OPNFV_NBD_DEV}" ]; then
+    sudo qemu-nbd -d "${OPNFV_NBD_DEV}" || true
+  fi
+}
+
 function cleanup_uefi {
   # Clean up Ubuntu boot entry if cfg01, kvm nodes online from previous deploy
   # shellcheck disable=SC2086
@@ -60,22 +164,45 @@ function cleanup_vms {
 function prepare_vms {
   local base_image=$1; shift
   local image_dir=$1; shift
+  local repos_pkgs_str=$1; shift # ^-sep list of repos, pkgs to install/rm
   local vnodes=("$@")
+  local image=base_image_opnfv_fuel.img

   cleanup_uefi
   cleanup_vms
   get_base_image "${base_image}" "${image_dir}"
+
+  rm -f "${image_dir}/${image%.*}"*
+  if [[ ! "${repos_pkgs_str}" =~ ^\^+$ ]]; then
+    IFS='^' read -r -a repos_pkgs <<< "${repos_pkgs_str}"
+    cp "${image_dir}/${base_image/*\/}" "${image_dir}/${image}"
+    mount_image "${image}" "${image_dir}"
+    apt_repos_pkgs_image "${repos_pkgs[@]:0:4}"
+    cleanup_mounts
+  else
+    ln -sf "${image_dir}/${base_image/*\/}" "${image_dir}/${image}"
+  fi
+
+  # CWD should be <mcp/scripts>
   # shellcheck disable=SC2016
   envsubst '${SALT_MASTER},${CLUSTER_DOMAIN}' < \
     user-data.template > user-data.sh

+  # Create config ISO and resize OS disk image for each foundation node VM
   for node in "${vnodes[@]}"; do
-    # create/prepare images
     ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" -u user-data.sh \
        -h "${node}" "${image_dir}/mcp_${node}.iso"
-    cp "${image_dir}/${base_image/*\/}" "${image_dir}/mcp_${node}.qcow2"
+    cp "${image_dir}/${image}" "${image_dir}/mcp_${node}.qcow2"
     qemu-img resize "${image_dir}/mcp_${node}.qcow2" 100G
   done
+
+  # VCP VMs base image specific changes
+  if [[ ! "${repos_pkgs_str}" =~ \^{3}$ ]] && [ -n "${repos_pkgs[*]:4}" ]; then
+    mount_image "${image}" "${image_dir}"
+    apt_repos_pkgs_image "${repos_pkgs[@]:4:4}"
+    cleanup_mounts
+    ln -sf "${image_dir}/${image}" "${image_dir}/${image%.*}_vcp.img"
+  fi
 }

 function create_networks {
@@ -100,6 +227,8 @@ function create_networks {

 function create_vms {
   local image_dir=$1; shift
+  # vnode data should be serialized with the following format:
+  # '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]'
   IFS='|' read -r -a vnodes <<< "$1"; shift
   local vnode_networks=("$@")

@@ -139,9 +268,7 @@ function create_vms {

 function update_mcpcontrol_network {
   # set static ip address for salt master node, MaaS node
-  # shellcheck disable=SC2155
   local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
-  # shellcheck disable=SC2155
   local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
   virsh net-update "mcpcontrol" add ip-dhcp-host \
     "<host mac='${cmac}' name='cfg01' ip='${SALT_MASTER}'/>" --live
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
index ab096f3..b719aa8 100755
--- a/mcp/scripts/salt.sh
+++ b/mcp/scripts/salt.sh
@@ -18,6 +18,8 @@ OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv"
 OPNFV_GIT_DIR="/root/opnfv"
 OPNFV_FUEL_DIR="/root/fuel"
 OPNFV_RDIR="reclass/classes/cluster/all-mcp-ocata-common"
+OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img"
+OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images"
 LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}"
 LOCAL_PDF_RECLASS=$1
 NODE_MASK='*'
@@ -34,6 +36,11 @@ if [ -n "${LOCAL_PDF_RECLASS}" ] && [ -f "${LOCAL_PDF_RECLASS}" ]; then
   rsync -e "ssh ${SSH_OPTS}" "${LOCAL_PDF_RECLASS}" \
     "${remote_tmp}${F_GIT_SUBD}/mcp/${OPNFV_RDIR}/opnfv/"
 fi
+local_vcp_img=$(dirname "${LOCAL_PDF_RECLASS}")/$(basename "${OPNFV_VCP_IMG}")
+if [ -e "${local_vcp_img}" ]; then
+  rsync -L -e "ssh ${SSH_OPTS}" "${local_vcp_img}" \
+    "${remote_tmp}${F_GIT_SUBD}/${OPNFV_VCP_IMG}"
+fi

 # ssh to cfg01
 # shellcheck disable=SC2086,2087
@@ -95,4 +102,9 @@ ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END
   salt -C "${NODE_MASK} and not cfg01*" pkg.upgrade refresh=False

   salt -C "${NODE_MASK} or cfg01*" state.sls ntp
+
+  if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then
+    mkdir -p "${OPNFV_VCP_DIR}"
+    mv "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" "${OPNFV_VCP_DIR}/"
+  fi
 SALT_INSTALL_END