Create local mirrors for offline installation

- switch to fuel-mirror [1]
- plugin developers can create packages.yaml
  file with names of required packages
- use custom configuration for fuel-bootstrap
  which always point to local repositories
- delay process of bootstrap image creation until
  mirrors will be ready

[1] https://github.com/openstack/fuel-mirror

Change-Id: Ie4a47c8cc13d9ded8c6b5fe48cdcbfeb1f6277a4
Signed-off-by: Michal Skalski <mskalski@mirantis.com>

12 files changed, 173 insertions, 44 deletions

diff --git a/build/Makefile b/build/Makefile
index 3d7dd634d..ab60f005f 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -53,7 +53,7 @@ export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py)
 
 export MIRROR_MOS_UBUNTU := $(shell echo "$(LATEST_MIRROR_ID_URL)" | cut -d'/' -f3)
 export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/8.0.target.txt" | head -1)
-export MIRROR_MOS_UBUNTU_ROOT := "/mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU)"
+export MIRROR_MOS_UBUNTU_ROOT := /mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU)
 
 export LATEST_TARGET_CENTOS := $(shell curl -sSf "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/os.target.txt" | head -1)
 export MIRROR_FUEL := "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos8.0-centos7-fuel/$(LATEST_TARGET_CENTOS)/x86_64"
diff --git a/build/config.mk b/build/config.mk
index 6b2f6a30e..eca426773 100644
--- a/build/config.mk
+++ b/build/config.mk
@@ -10,6 +10,8 @@
 
 FUEL_MAIN_REPO := https://github.com/openstack/fuel-main
 FUEL_MAIN_TAG = stable/8.0
+MOS_VERSION = 8.0
+OPENSTACK_VERSION = liberty-8.0
 
 DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64
 DOCKER_TAG := docker-latest
diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
index 4ac51476e..b7ef6d3a7 100644
--- a/build/docker/Dockerfile
+++ b/build/docker/Dockerfile
@@ -25,7 +25,7 @@ RUN echo "Defaults env_keep += \"ftp_proxy http_proxy https_proxy no_proxy RSYNC
 # Keeping PWD is needed to build as root
 RUN echo "Defaults env_keep += \"PWD\"" > /etc/sudoers.d/keep-pwd
 # Keeping variables for ISO build
-RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL\"" > /etc/sudoers.d/keep-mos
+RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL LATEST_TARGET_UBUNTU\"" > /etc/sudoers.d/keep-mos
 RUN chmod 0440 /etc/sudoers.d/open-sudo
 RUN chmod 0440 /etc/sudoers.d/keep-proxies
 RUN chmod 0440 /etc/sudoers.d/keep-pwd
diff --git a/build/docker/runcontext b/build/docker/runcontext
index 2df765868..9f077764e 100755
--- a/build/docker/runcontext
+++ b/build/docker/runcontext
@@ -111,7 +111,7 @@ if [ -n "$CACHEBASE" ]; then
     fi
 fi
 
-RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT"
+RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL -e LATEST_TARGET_UBUNTU -u $USER_ID:$GROUP_ID -w $PWD -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT"
 
 # Passing "debug" puts up an interactive bash shell
 if [ "$1" == "debug" ]; then
diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile
index 21d47f4f3..ee27adb49 100644
--- a/build/f_isoroot/Makefile
+++ b/build/f_isoroot/Makefile
@@ -10,6 +10,7 @@
 
 # Add Fuel plugin build targets here
 PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_qemupluginbuild f_ovs-nsh-dpdk-pluginbuild
+export PLUGINS
 #NON_8-0_REBASED_PLUGINS = f_bgpvpn-pluginbuild
 
 # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins
@@ -17,12 +18,11 @@ PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsper
 # Temporarilu disabling f_repobuild that currently have some issues on
 # Fuel 8.
 ifdef BUILD_FUEL_PLUGINS
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS)
+export BUILD_FUEL_PLUGINS
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
 $(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS))
 else
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS)
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
 endif
 SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
 
diff --git a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
index a5ca56c64..b096a7812 100755
--- a/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
+++ b/build/f_isoroot/f_bootstrap/bootstrap_admin_node.sh
@@ -414,6 +414,16 @@ rmdir /var/log/remote && ln -s /var/log/docker-logs/remote /var/log/remote
 dockerctl check || fail
 bash /etc/rc.local
 
+### OPNFV addition BEGIN
+shopt -s nullglob
+for script in /opt/opnfv/bootstrap/post.d/*.sh
+do
+  echo "Post script: $script" >> /root/post.log 2>&1
+  $script >> /root/post.log 2>&1
+done
+shopt -u nullglob
+### OPNFV addition END
+
 if [ "`get_bootstrap_flavor`" = "ubuntu" ]; then
   if [ "`get_bootstrap_skip`" = "False" ]; then
     build_ubuntu_bootstrap bs_status || true
@@ -426,15 +436,6 @@ else
   bs_status=3
 fi
 
-### OPNFV addition BEGIN
-shopt -s nullglob
-for script in /opt/opnfv/bootstrap/post.d/*.sh
-do
-  echo "Post script: $script" >> /root/post.log 2>&1
-  $script >> /root/post.log 2>&1
-done
-shopt -u nullglob
-### OPNFV addition END
 
 # Enable updates repository
 cat > /etc/yum.repos.d/mos${FUEL_RELEASE}-updates.repo << EOF
diff --git a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
index 427a55add..8dad89397 100755
--- a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
+++ b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh
@@ -16,4 +16,9 @@ if [ $? -ne 0 ]; then
   echo "Error removing /opt/opnfv/nailgun directory!"
   exit 1
 fi
+mv /opt/opnfv/fuel_bootstrap_cli.yaml /etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml
+if [ $? -ne 0 ]; then
+  echo "Error moving bootstrap image configuration!"
+  exit 1
+fi
 echo "Done installing pre-build repo"
diff --git a/build/f_isoroot/f_odlpluginbuild/packages.yaml b/build/f_isoroot/f_odlpluginbuild/packages.yaml
new file mode 100644
index 000000000..7ef537f31
--- /dev/null
+++ b/build/f_isoroot/f_odlpluginbuild/packages.yaml
@@ -0,0 +1,3 @@
+packages:
+  - "openjdk-7-jre-headless"
+  - "acct"
diff --git a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt b/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt
deleted file mode 100644
index f0d06681e..000000000
--- a/build/f_isoroot/f_odlpluginbuild/requirements-deb.txt
+++ /dev/null
@@ -1 +0,0 @@
-openjdk-7-jre-headless
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
index 1fc703031..966da3634 100644
--- a/build/f_isoroot/f_repobuild/Makefile
+++ b/build/f_isoroot/f_repobuild/Makefile
@@ -10,54 +10,48 @@
 
 SHELL := /bin/bash
 TOP := $(shell pwd)
-RSYNC_HOST := $(shell ./select_ubuntu_repo.sh)
-DEPENDENCIES_FILE := /etc/fuel-createmirror/requirements-deb.txt
+UBUNTU_DIR := $(shell dirname $(MIRROR_UBUNTU_ROOT))
 
+include ../../config.mk
+
+export MOS_VERSION
+export OPENSTACK_VERSION
 
 .PHONY: all
 all: nailgun
 
 nailgun:
-#	If RSYNC_CONNECT_PROG is set we need to do some magic to copy the
-#	keys. Make sure to have the username set in the SSH_CONNECT_PROG
-#	as well!
-	@if [ -n "${RSYNC_CONNECT_PROG}" -a ! -d /root/.ssh ]; then \
-		sudo mkdir -p /root/.ssh; \
-		test -d ${HOME}/.ssh && sudo find ${HOME}/.ssh -maxdepth 1 -type f -exec cp {} /root/.ssh \; ; \
-		sudo bash -c "echo StrictHostKeyChecking=no > /root/.ssh/config"; \
-		sudo chmod 700 /root/.ssh; \
-	fi
-	sudo apt-get install -y rsync python python-yaml dpkg-dev openssl
-	rm -rf tmpiso tmpdir
-	mkdir tmpiso
-	fuseiso ${ISOCACHE} tmpiso
-	cp tmpiso/ubuntu/pool/main/f/fuel-createmirror/fuel-createmirror_*.deb .
-	fusermount -u tmpiso
-	rm -rf tmpiso
-	sudo dpkg -i fuel-createmirror_*.deb
-	sudo sed -i 's/DOCKER_MODE=true/DOCKER_MODE=false/' /etc/fuel-createmirror/common.cfg
-	sudo sed -i 's/DEBUG="no"/DEBUG="yes"/' /etc/fuel-createmirror/ubuntu.cfg
-	sudo sed -i "s/MIRROR_UBUNTU_HOST=\".*\"/MIRROR_UBUNTU_HOST=\"$(RSYNC_HOST)\"/" /etc/fuel-createmirror/common.cfg
-	# Add dependencies from plugins. With awk help try add only unique packages.
-	sudo bash -c 'find ../ -name requirements-deb.txt -print0 | xargs -0 -I% awk '\''NR == FNR {first[$$0];next}; ! ($$0 in first)'\'' "${DEPENDENCIES_FILE}" "%" >> "${DEPENDENCIES_FILE}"'
+	sudo apt-get install -y git libxml2-dev libxslt-dev python-dev  python-pip libz-dev libyaml-dev createrepo python-yaml
 	rm -Rf nailgun
-	sudo mkdir -p /var/www
-	sudo su - -c /opt/fuel-createmirror-*/fuel-createmirror
+	sudo mkdir -p /var/www/nailgun
+	git clone -b $(FUEL_MAIN_TAG) https://github.com/openstack/fuel-mirror
+	sudo pip install -U -r ./fuel-mirror/requirements.txt
+	sudo pip install ./fuel-mirror
+	sudo pip install ./fuel-mirror/contrib/fuel_mirror
+	./opnfv_mirror_conf.py
+	sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group mos ubuntu --pattern=ubuntu
 	sudo chmod -R 755 /var/www/nailgun
 	cp -Rp /var/www/nailgun .
+	mv nailgun/mirrors$(MIRROR_MOS_UBUNTU_ROOT) nailgun/mirrors/mos-repos/ubuntu/$(MOS_VERSION)
+	rmdir nailgun/mirrors/mos-repos/ubuntu/snapshots
+	if [ "$(UBUNTU_DIR)" != "/" ]; then \
+	  mv nailgun/mirrors$(UBUNTU_DIR)/ubuntu naligun/mirros/ubuntu;\
+	  rm -rf nailgun/mirrors$(UBUNTU_DIR);\
+	fi
 	# Store artifact in cache straight away if caching is enabled
 	# (no .cacheid will be present unless this is a cached build)
 	test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
 
 .PHONY: clean
 clean:
-	@rm -rf ../release/opnfv/nailgun nailgun fuel-createmirror_6.1*.deb
+	@rm -rf ../release/opnfv/nailgun nailgun fuel-mirror
 
 .PHONY: release
 release:nailgun
 	@rm -Rf ../release/opnfv/nailgun
 	@mkdir -p ../release/opnfv
 	@cp -Rp nailgun ../release/opnfv/nailgun
+	@cp fuel_bootstrap_cli.yaml ../release/opnfv/
 
 #############################################################################
 # Cache operations - only used when building through ci/build.sh
diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
new file mode 100644
index 000000000..28f6427f7
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
@@ -0,0 +1,75 @@
+---
+  root_ssh_authorized_file: /root/.ssh/id_rsa.pub
+  extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8"
+  extra_dirs:
+    - /usr/share/fuel_bootstrap_cli/files/trusty
+  output_dir: /tmp/
+  kernel_flavor: linux-image-generic-lts-trusty
+  packages:
+    - vim
+    - mc
+    - openssh-client
+    - openssh-server
+    - ntp
+    - mcollective
+    - nailgun-agent
+    - nailgun-mcagents
+    - network-checker
+    - fuel-agent
+    - ubuntu-minimal
+    - live-boot
+    - live-boot-initramfs-tools
+    - wget
+    - linux-firmware
+    - linux-firmware-nonfree
+    - linux-headers-generic
+    - hpsa-dkms
+    - i40e-dkms
+    - xz-utils
+    - squashfs-tools
+    - msmtp-mta
+  bootstrap_images_dir: /var/www/nailgun/bootstraps
+  active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap
+  flavor: ubuntu
+  http_proxy: ""
+  https_proxy: ""
+  repos:
+    - name: ubuntu
+      section: "main universe multiverse"
+      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+      priority:
+      suite: trusty
+      type: deb
+    - name: ubuntu-updates
+      section: "main universe multiverse"
+      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+      priority:
+      suite: trusty-updates
+      type: deb
+    - name: ubuntu-security
+      section: "main universe multiverse"
+      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+      priority:
+      suite: trusty-security
+      type: deb
+    - name: mos
+      section: "main restricted"
+      uri: "http://127.0.0.1:8080/ubuntu/x86_64"
+      priority: 1050
+      suite: mos8.0
+      type: deb
+    - name: mos-updates
+      section: "main restricted"
+      uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+      priority: 1050
+      suite: mos8.0-updates
+      type: deb
+    - name: mos-security
+      section: "main restricted"
+      uri: "http://127.0.0.1:8080/mirrors/mos-repos/ubuntu/8.0"
+      priority: 1050
+      suite: mos8.0-security
+      type: deb
+  skip_default_img_build: true
+  direct_repo_addresses:
+    - "127.0.0.1"
diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
new file mode 100755
index 000000000..9abd0242b
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import yaml
+
+current_snapshot =  os.environ["LATEST_TARGET_UBUNTU"]
+mos_version = os.environ['MOS_VERSION']
+openstack_version = os.environ['OPENSTACK_VERSION']
+mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU']
+mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT']
+mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL']
+if os.environ.get('BUILD_FUEL_PLUGINS'):
+  plugins = os.environ['BUILD_FUEL_PLUGINS']
+else:
+  plugins = os.environ['PLUGINS']
+
+
+configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read()
+conf = yaml.load(configuration_file)
+conf['pattern_dir'] = '.'
+conf['openstack_version'] = openstack_version
+conf['mos_version'] = mos_version
+
+with open('opnfv-config.yaml', 'w') as outfile:
+  outfile.write( yaml.dump(conf, default_flow_style=False) )
+
+pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
+pattern = yaml.load(pattern_file)
+pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
+pattern['ubuntu_baseurl'] = mirror_ubuntu
+for group in pattern['groups']['mos']:
+  group['uri'] = pattern['mos_baseurl']
+for group in pattern['groups']['ubuntu']:
+  group['uri'] = pattern['ubuntu_baseurl']
+
+for plugin in plugins.split():
+  path = "../{}/packages.yaml".format(plugin)
+  if os.path.isfile(path):
+    f = open(path).read()
+    plugin_yaml = yaml.load(f)
+    plugin_set = set(plugin_yaml['packages'])
+    main_set = set(pattern['packages'])
+    new_packages = plugin_set - main_set
+    print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages))
+    pattern['packages'] = pattern['packages'] + list(new_packages)
+
+pattern['requirements']['ubuntu'] = pattern['packages']
+
+with open('ubuntu.yaml', 'w') as outfile:
+  outfile.write( yaml.safe_dump(pattern, default_flow_style=False) )