summaryrefslogtreecommitdiffstats
path: root/build/f_isoroot/f_repobuild
diff options
context:
space:
mode:
authorMichal Skalski <mskalski@mirantis.com>2016-01-28 15:28:07 +0100
committerStefan Berg <sfb@consultron.com>2016-01-29 14:41:57 +0100
commit65c164021cb356b3d4e6888488fb62c3395da4f5 (patch)
tree8225bb2225c48f450b66eebd1682cd122b5c2cb4 /build/f_isoroot/f_repobuild
parent5aa0078848414d6492118f9c6d533c5de3d70bf6 (diff)
Create local mirrors for offline installation
- switch to fuel-mirror [1] - plugin developers can create packages.yaml file with names of required packages - use custom configuration for fuel-bootstrap which always point to local repositories - delay process of bootstrap image creation until mirrors will be ready [1] https://github.com/openstack/fuel-mirror Change-Id: Ie4a47c8cc13d9ded8c6b5fe48cdcbfeb1f6277a4 Signed-off-by: Michal Skalski <mskalski@mirantis.com> (cherry picked from commit f595c31b29147efb1410756b491ff7bf48d256ec)
Diffstat (limited to 'build/f_isoroot/f_repobuild')
-rw-r--r--build/f_isoroot/f_repobuild/Makefile48
-rw-r--r--build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml75
-rwxr-xr-xbuild/f_isoroot/f_repobuild/opnfv_mirror_conf.py50
3 files changed, 146 insertions, 27 deletions
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
index 1fc703031..966da3634 100644
--- a/build/f_isoroot/f_repobuild/Makefile
+++ b/build/f_isoroot/f_repobuild/Makefile
@@ -10,54 +10,48 @@
SHELL := /bin/bash
TOP := $(shell pwd)
-RSYNC_HOST := $(shell ./select_ubuntu_repo.sh)
-DEPENDENCIES_FILE := /etc/fuel-createmirror/requirements-deb.txt
+UBUNTU_DIR := $(shell dirname $(MIRROR_UBUNTU_ROOT))
+include ../../config.mk
+
+export MOS_VERSION
+export OPENSTACK_VERSION
.PHONY: all
all: nailgun
nailgun:
-# If RSYNC_CONNECT_PROG is set we need to do some magic to copy the
-# keys. Make sure to have the username set in the SSH_CONNECT_PROG
-# as well!
- @if [ -n "${RSYNC_CONNECT_PROG}" -a ! -d /root/.ssh ]; then \
- sudo mkdir -p /root/.ssh; \
- test -d ${HOME}/.ssh && sudo find ${HOME}/.ssh -maxdepth 1 -type f -exec cp {} /root/.ssh \; ; \
- sudo bash -c "echo StrictHostKeyChecking=no > /root/.ssh/config"; \
- sudo chmod 700 /root/.ssh; \
- fi
- sudo apt-get install -y rsync python python-yaml dpkg-dev openssl
- rm -rf tmpiso tmpdir
- mkdir tmpiso
- fuseiso ${ISOCACHE} tmpiso
- cp tmpiso/ubuntu/pool/main/f/fuel-createmirror/fuel-createmirror_*.deb .
- fusermount -u tmpiso
- rm -rf tmpiso
- sudo dpkg -i fuel-createmirror_*.deb
- sudo sed -i 's/DOCKER_MODE=true/DOCKER_MODE=false/' /etc/fuel-createmirror/common.cfg
- sudo sed -i 's/DEBUG="no"/DEBUG="yes"/' /etc/fuel-createmirror/ubuntu.cfg
- sudo sed -i "s/MIRROR_UBUNTU_HOST=\".*\"/MIRROR_UBUNTU_HOST=\"$(RSYNC_HOST)\"/" /etc/fuel-createmirror/common.cfg
- # Add dependencies from plugins. With awk help try add only unique packages.
- sudo bash -c 'find ../ -name requirements-deb.txt -print0 | xargs -0 -I% awk '\''NR == FNR {first[$$0];next}; ! ($$0 in first)'\'' "${DEPENDENCIES_FILE}" "%" >> "${DEPENDENCIES_FILE}"'
+ sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml
rm -Rf nailgun
- sudo mkdir -p /var/www
- sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror
+ sudo mkdir -p /var/www/nailgun
+ git clone -b $(FUEL_MAIN_TAG) https://github.com/openstack/fuel-mirror
+ sudo pip install -U -r ./fuel-mirror/requirements.txt
+ sudo pip install ./fuel-mirror
+ sudo pip install ./fuel-mirror/contrib/fuel_mirror
+ ./opnfv_mirror_conf.py
+ sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group mos ubuntu --pattern=ubuntu
sudo chmod -R 755 /var/www/nailgun
cp -Rp /var/www/nailgun .
+ mv nailgun/mirrors$(MIRROR_MOS_UBUNTU_ROOT) nailgun/mirrors/mos-repos/ubuntu/$(MOS_VERSION)
+ rmdir nailgun/mirrors/mos-repos/ubuntu/snapshots
+ if [ "$(UBUNTU_DIR)" != "/" ]; then \
+ mv nailgun/mirrors$(UBUNTU_DIR)/ubuntu naligun/mirros/ubuntu;\
+ rm -rf nailgun/mirrors$(UBUNTU_DIR);\
+ fi
# Store artifact in cache straight away if caching is enabled
# (no .cacheid will be present unless this is a cached build)
test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
.PHONY: clean
clean:
- @rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb
+ @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror
.PHONY: release
release:nailgun
@rm -Rf ../release/opnfv/nailgun
@mkdir -p ../release/opnfv
@cp -Rp nailgun ../release/opnfv/nailgun
+ @cp fuel_bootstrap_cli.yaml ../release/opnfv/
#############################################################################
# Cache operations - only used when building through ci/build.sh
diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
new file mode 100644
index 000000000..28f6427f7
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
@@ -0,0 +1,75 @@
+---
+ root_ssh_authorized_file: /root/.ssh/id_rsa.pub
+ extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8"
+ extra_dirs:
+ - /usr/share/fuel_bootstrap_cli/files/trusty
+ output_dir: /tmp/
+ kernel_flavor: linux-image-generic-lts-trusty
+ packages:
+ - vim
+ - mc
+ - openssh-client
+ - openssh-server
+ - ntp
+ - mcollective
+ - nailgun-agent
+ - nailgun-mcagents
+ - network-checker
+ - fuel-agent
+ - ubuntu-minimal
+ - live-boot
+ - live-boot-initramfs-tools
+ - wget
+ - linux-firmware
+ - linux-firmware-nonfree
+ - linux-headers-generic
+ - hpsa-dkms
+ - i40e-dkms
+ - xz-utils
+ - squashfs-tools
+ - msmtp-mta
+ bootstrap_images_dir: /var/www/nailgun/bootstraps
+ active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap
+ flavor: ubuntu
+ http_proxy: ""
+ https_proxy: ""
+ repos:
+ - name: ubuntu
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty
+ type: deb
+ - name: ubuntu-updates
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty-updates
+ type: deb
+ - name: ubuntu-security
+ section: "main universe multiverse"
+ uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+ priority:
+ suite: trusty-security
+ type: deb
+ - name: mos
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/ubuntu/x86_64"
+ priority: 1050
+ suite: mos8.0
+ type: deb
+ - name: mos-updates
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+ priority: 1050
+ suite: mos8.0-updates
+ type: deb
+ - name: mos-security
+ section: "main restricted"
+ uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+ priority: 1050
+ suite: mos8.0-security
+ type: deb
+ skip_default_img_build: true
+ direct_repo_addresses:
+ - "127.0.0.1"
diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
new file mode 100755
index 000000000..9abd0242b
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import yaml
+
+current_snapshot = os.environ["LATEST_TARGET_UBUNTU"]
+mos_version = os.environ['MOS_VERSION']
+openstack_version = os.environ['OPENSTACK_VERSION']
+mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU']
+mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT']
+mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL']
+if os.environ.get('BUILD_FUEL_PLUGINS'):
+ plugins = os.environ['BUILD_FUEL_PLUGINS']
+else:
+ plugins = os.environ['PLUGINS']
+
+
+configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read()
+conf = yaml.load(configuration_file)
+conf['pattern_dir'] = '.'
+conf['openstack_version'] = openstack_version
+conf['mos_version'] = mos_version
+
+with open('opnfv-config.yaml', 'w') as outfile:
+ outfile.write( yaml.dump(conf, default_flow_style=False) )
+
+pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
+pattern = yaml.load(pattern_file)
+pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
+pattern['ubuntu_baseurl'] = mirror_ubuntu
+for group in pattern['groups']['mos']:
+ group['uri'] = pattern['mos_baseurl']
+for group in pattern['groups']['ubuntu']:
+ group['uri'] = pattern['ubuntu_baseurl']
+
+for plugin in plugins.split():
+ path = "../{}/packages.yaml".format(plugin)
+ if os.path.isfile(path):
+ f = open(path).read()
+ plugin_yaml = yaml.load(f)
+ plugin_set = set(plugin_yaml['packages'])
+ main_set = set(pattern['packages'])
+ new_packages = plugin_set - main_set
+ print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages))
+ pattern['packages'] = pattern['packages'] + list(new_packages)
+
+pattern['requirements']['ubuntu'] = pattern['packages']
+
+with open('ubuntu.yaml', 'w') as outfile:
+ outfile.write( yaml.safe_dump(pattern, default_flow_style=False) )