From 65c164021cb356b3d4e6888488fb62c3395da4f5 Mon Sep 17 00:00:00 2001 From: Michal Skalski Date: Thu, 28 Jan 2016 15:28:07 +0100 Subject: Create local mirrors for offline installation - switch to fuel-mirror [1] - plugin developers can create packages.yaml file with names of required packages - use custom configuration for fuel-bootstrap which always point to local repositories - delay process of bootstrap image creation until mirrors will be ready [1] https://github.com/openstack/fuel-mirror Change-Id: Ie4a47c8cc13d9ded8c6b5fe48cdcbfeb1f6277a4 Signed-off-by: Michal Skalski (cherry picked from commit f595c31b29147efb1410756b491ff7bf48d256ec) --- build/f_isoroot/Makefile | 8 +-- .../f_isoroot/f_bootstrap/bootstrap_admin_node.sh | 9 +-- .../f_bootstrap/post-scripts/03_install_repo.sh | 5 ++ build/f_isoroot/f_odlpluginbuild/packages.yaml | 3 + .../f_odlpluginbuild/requirements-deb.txt | 1 - build/f_isoroot/f_repobuild/Makefile | 48 ++++++-------- .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 75 ++++++++++++++++++++++ build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 50 +++++++++++++++ 8 files changed, 163 insertions(+), 36 deletions(-) create mode 100644 build/f_isoroot/f_odlpluginbuild/packages.yaml delete mode 100644 build/f_isoroot/f_odlpluginbuild/requirements-deb.txt create mode 100644 build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py (limited to 'build/f_isoroot') diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile index 703d85a0b..c14c09649 100644 --- a/build/f_isoroot/Makefile +++ b/build/f_isoroot/Makefile @@ -10,17 +10,17 @@ # Add Fuel plugin build targets here PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild +export PLUGINS #NON_8-0_REBASED_PLUGINS = f_bgpvpn-pluginbuild # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins # indicated therein. ifdef BUILD_FUEL_PLUGINS -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS) +export BUILD_FUEL_PLUGINS +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) $(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) else -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS) +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) endif SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh index 35d107ab4..8bf1bb165 100755 --- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh +++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh @@ -277,10 +277,6 @@ rmdir /var/log/remote && ln -s /var/log/docker-logs/remote /var/log/remote dockerctl check || fail bash /etc/rc.local -if [ "`get_bootstrap_flavor`" = "ubuntu" ]; then - build_ubuntu_bootstrap || true -fi - ### OPNFV addition BEGIN shopt -s nullglob for script in /opt/opnfv/bootstrap/post.d/*.sh @@ -291,6 +287,11 @@ done shopt -u nullglob ### OPNFV addition END +if [ "`get_bootstrap_flavor`" = "ubuntu" ]; then + build_ubuntu_bootstrap || true +fi + + # Enable updates repository cat > /etc/yum.repos.d/mos${FUEL_RELEASE}-updates.repo << EOF [mos${FUEL_RELEASE}-updates] diff --git a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh index 427a55add..8dad89397 100755 --- a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh +++ b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh @@ -16,4 +16,9 @@ if [ $? -ne 0 ]; then echo "Error removing /opt/opnfv/nailgun directory!" exit 1 fi +mv /opt/opnfv/fuel_bootstrap_cli.yaml /etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml +if [ $? -ne 0 ]; then + echo "Error moving bootstrap image configuration!" + exit 1 +fi echo "Done installing pre-build repo" diff --git a/build/f_isoroot/f_odlpluginbuild/packages.yaml b/build/f_isoroot/f_odlpluginbuild/packages.yaml new file mode 100644 index 000000000..7ef537f31 --- /dev/null +++ b/build/f_isoroot/f_odlpluginbuild/packages.yaml @@ -0,0 +1,3 @@ +packages: + - "openjdk-7-jre-headless" + - "acct" diff --git a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt b/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt deleted file mode 100644 index f0d06681e..000000000 --- a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt +++ /dev/null @@ -1 +0,0 @@ -openjdk-7-jre-headless diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile index 1fc703031..966da3634 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -10,54 +10,48 @@ SHELL := /bin/bash TOP := $(shell pwd) -RSYNC_HOST := $(shell ./select_ubuntu_repo.sh) -DEPENDENCIES_FILE := /etc/fuel-createmirror/requirements-deb.txt +UBUNTU_DIR := $(shell dirname $(MIRROR_UBUNTU_ROOT)) +include ../../config.mk + +export MOS_VERSION +export OPENSTACK_VERSION .PHONY: all all: nailgun nailgun: -# If RSYNC_CONNECT_PROG is set we need to do some magic to copy the -# keys. Make sure to have the username set in the SSH_CONNECT_PROG -# as well! - @if [ -n "${RSYNC_CONNECT_PROG}" -a ! -d /root/.ssh ]; then \ - sudo mkdir -p /root/.ssh; \ - test -d ${HOME}/.ssh && sudo find ${HOME}/.ssh -maxdepth 1 -type f -exec cp {} /root/.ssh \; ; \ - sudo bash -c "echo StrictHostKeyChecking=no > /root/.ssh/config"; \ - sudo chmod 700 /root/.ssh; \ - fi - sudo apt-get install -y rsync python python-yaml dpkg-dev openssl - rm -rf tmpiso tmpdir - mkdir tmpiso - fuseiso ${ISOCACHE} tmpiso - cp tmpiso/ubuntu/pool/main/f/fuel-createmirror/fuel-createmirror_*.deb . - fusermount -u tmpiso - rm -rf tmpiso - sudo dpkg -i fuel-createmirror_*.deb - sudo sed -i 's/DOCKER_MODE=true/DOCKER_MODE=false/' /etc/fuel-createmirror/common.cfg - sudo sed -i 's/DEBUG="no"/DEBUG="yes"/' /etc/fuel-createmirror/ubuntu.cfg - sudo sed -i "s/MIRROR_UBUNTU_HOST=\".*\"/MIRROR_UBUNTU_HOST=\"$(RSYNC_HOST)\"/" /etc/fuel-createmirror/common.cfg - # Add dependencies from plugins. With awk help try add only unique packages. - sudo bash -c 'find ../ -name requirements-deb.txt -print0 | xargs -0 -I% awk '\''NR == FNR {first[$$0];next}; ! ($$0 in first)'\'' "${DEPENDENCIES_FILE}" "%" >> "${DEPENDENCIES_FILE}"' + sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml rm -Rf nailgun - sudo mkdir -p /var/www - sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror + sudo mkdir -p /var/www/nailgun + git clone -b $(FUEL_MAIN_TAG) https://github.com/openstack/fuel-mirror + sudo pip install -U -r ./fuel-mirror/requirements.txt + sudo pip install ./fuel-mirror + sudo pip install ./fuel-mirror/contrib/fuel_mirror + ./opnfv_mirror_conf.py + sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group mos ubuntu --pattern=ubuntu sudo chmod -R 755 /var/www/nailgun cp -Rp /var/www/nailgun . + mv nailgun/mirrors$(MIRROR_MOS_UBUNTU_ROOT) nailgun/mirrors/mos-repos/ubuntu/$(MOS_VERSION) + rmdir nailgun/mirrors/mos-repos/ubuntu/snapshots + if [ "$(UBUNTU_DIR)" != "/" ]; then \ + mv nailgun/mirrors$(UBUNTU_DIR)/ubuntu naligun/mirros/ubuntu;\ + rm -rf nailgun/mirrors$(UBUNTU_DIR);\ + fi # Store artifact in cache straight away if caching is enabled # (no .cacheid will be present unless this is a cached build) test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 .PHONY: clean clean: - @rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb + @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror .PHONY: release release:nailgun @rm -Rf ../release/opnfv/nailgun @mkdir -p ../release/opnfv @cp -Rp nailgun ../release/opnfv/nailgun + @cp fuel_bootstrap_cli.yaml ../release/opnfv/ ############################################################################# # Cache operations - only used when building through ci/build.sh diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml new file mode 100644 index 000000000..28f6427f7 --- /dev/null +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml @@ -0,0 +1,75 @@ +--- + root_ssh_authorized_file: /root/.ssh/id_rsa.pub + extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" + extra_dirs: + - /usr/share/fuel_bootstrap_cli/files/trusty + output_dir: /tmp/ + kernel_flavor: linux-image-generic-lts-trusty + packages: + - vim + - mc + - openssh-client + - openssh-server + - ntp + - mcollective + - nailgun-agent + - nailgun-mcagents + - network-checker + - fuel-agent + - ubuntu-minimal + - live-boot + - live-boot-initramfs-tools + - wget + - linux-firmware + - linux-firmware-nonfree + - linux-headers-generic + - hpsa-dkms + - i40e-dkms + - xz-utils + - squashfs-tools + - msmtp-mta + bootstrap_images_dir: /var/www/nailgun/bootstraps + active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap + flavor: ubuntu + http_proxy: "" + https_proxy: "" + repos: + - name: ubuntu + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty + type: deb + - name: ubuntu-updates + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-updates + type: deb + - name: ubuntu-security + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-security + type: deb + - name: mos + section: "main restricted" + uri: "http://127.0.0.1:8080/ubuntu/x86_64" + priority: 1050 + suite: mos8.0 + type: deb + - name: mos-updates + section: "main restricted" + uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0" + priority: 1050 + suite: mos8.0-updates + type: deb + - name: mos-security + section: "main restricted" + uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0" + priority: 1050 + suite: mos8.0-security + type: deb + skip_default_img_build: true + direct_repo_addresses: + - "127.0.0.1" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py new file mode 100755 index 000000000..9abd0242b --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import os +import yaml + +current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] +mos_version = os.environ['MOS_VERSION'] +openstack_version = os.environ['OPENSTACK_VERSION'] +mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] +mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] +mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] +if os.environ.get('BUILD_FUEL_PLUGINS'): + plugins = os.environ['BUILD_FUEL_PLUGINS'] +else: + plugins = os.environ['PLUGINS'] + + +configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() +conf = yaml.load(configuration_file) +conf['pattern_dir'] = '.' +conf['openstack_version'] = openstack_version +conf['mos_version'] = mos_version + +with open('opnfv-config.yaml', 'w') as outfile: + outfile.write( yaml.dump(conf, default_flow_style=False) ) + +pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() +pattern = yaml.load(pattern_file) +pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) +pattern['ubuntu_baseurl'] = mirror_ubuntu +for group in pattern['groups']['mos']: + group['uri'] = pattern['mos_baseurl'] +for group in pattern['groups']['ubuntu']: + group['uri'] = pattern['ubuntu_baseurl'] + +for plugin in plugins.split(): + path = "../{}/packages.yaml".format(plugin) + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) + plugin_set = set(plugin_yaml['packages']) + main_set = set(pattern['packages']) + new_packages = plugin_set - main_set + print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) + pattern['packages'] = pattern['packages'] + list(new_packages) + +pattern['requirements']['ubuntu'] = pattern['packages'] + +with open('ubuntu.yaml', 'w') as outfile: + outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) -- cgit 1.2.3-korg