Adding script to build samplevnf VM images

Change-Id: I1238aa72f178fb8744fdea688704ef7ff22c9c75
Signed-off-by: Deepak S <deepak.s@linux.intel.com>

2 files changed, 395 insertions, 0 deletions

diff --git a/tools/samplevnf-img-dpdk-samplevnf-modify b/tools/samplevnf-img-dpdk-samplevnf-modify
new file mode 100755
index 00000000..a79fb216
--- /dev/null
+++ b/tools/samplevnf-img-dpdk-samplevnf-modify
@@ -0,0 +1,194 @@
+#!/bin/bash
+
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# samplevnf-img-dpdk-nsb-modify - download and modify a Ubuntu cloud image
+#
+# The actual customization is done by a script passed with an absolute path as
+# the only single argument. The command needs to be invoked as sudo
+#
+# Example invocation:
+# samplevnf-img-dpdk-nsb-modify /home/samplevnf/tools/ubuntu-server-cloudimg-nsb-modify.sh
+#
+# Warning: the script will create files by default in:
+#   /tmp/workspace/samplevnf
+# the files will be owned by root!
+#
+# TODO: image resize is needed if the base image is too small
+#
+set -e
+set -x
+
+die() {
+    echo "error: $1" >&2
+    exit 1
+}
+
+test $# -eq 1 -o $# -eq 2 || die "no image specific script as argument"
+test $(id -u) -eq 0 || die "should invoke using sudo"
+
+cmd=$1
+RELEASE=$2
+test -x $cmd
+mountdir="/mnt/samplevnf"
+workspace=${WORKSPACE:-"/tmp/workspace/samplevnf"}
+host=${HOST:-"cloud-images.ubuntu.com"}
+release=${RELEASE:-"xenial"}
+boot_mode="disk1"
+YARD_IMG_ARCH="amd64"
+if [[ "${YARD_IMG_ARCH}" = "arm64" ]]; then
+    boot_mode="uefi1"
+fi
+
+image_path="${release}/current/${release}-server-cloudimg-${YARD_IMG_ARCH}-${boot_mode}.img"
+image_url=${IMAGE_URL:-"https://${host}/${image_path}"}
+sha256sums_path="${release}/current/SHA256SUMS"
+sha256sums_url=${SHA256SUMS_URL:-"https://${host}/${sha256sums_path}"}
+
+imgfile="${workspace}/samplevnf-image.img"
+raw_imgfile_basename="samplevnf-${release}-server.raw"
+raw_imgfile="${workspace}/${raw_imgfile_basename}"
+filename=$(basename $image_url)
+YARD_IMG_ARCH="amd64" #Neha added this just for testing
+apt-get install -y parted
+
+# download and checksum base image, conditionally if local copy is outdated
+download() {
+    test -d $workspace || mkdir -p $workspace
+    cd $workspace
+    rm -f SHA256SUMS # always download the checksum file to a detect stale image
+    wget $sha256sums_url
+    test -e $filename || wget -nc --progress=dot:giga $image_url
+    grep $filename SHA256SUMS | sha256sum -c ||
+    if [ $? -ne 0 ]; then
+        rm $filename
+        wget -nc --progress=dot:giga $image_url
+        grep $filename SHA256SUMS | sha256sum -c
+    fi
+
+    for i in $(seq 0 9); do
+        [ -a /dev/loop$i ] || mknod -m 660 /dev/loop$i b 7 $i
+    done
+
+    qemu-img convert $filename $raw_imgfile
+    cd -
+}
+
+# mount image
+setup() {
+#    qemu-img resize $raw_imgfile +5GB
+    mkdir -p $mountdir
+
+    loopdevice=$(kpartx -l $raw_imgfile | head -1 | cut -f1 -d ' ')
+
+    kpartx -av $raw_imgfile
+
+    # for trouble shooting
+    sleep 2
+    dmsetup ls
+    fuser -c /dev/loop0
+    fuser -f /dev/loop0
+    parted -l /dev/${loopdevice:0:5} || true
+    mount /dev/mapper/$loopdevice $mountdir
+    mount -t proc none $mountdir/proc
+
+    sudo resize2fs /dev/mapper/$loopdevice
+    cp $cmd $mountdir/$(basename $cmd)
+    YARD_IMG_ARCH="amd64" #Neha added this just for testing
+    if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
+        cp /usr/bin/qemu-aarch64-static $mountdir/usr/bin
+    fi
+}
+
+# modify image running a script using in a chrooted environment
+modify() {
+    # resolv.conf does not exist in base image, pass nameserver value from host
+    nameserver_ip=$(grep -m 1 '^nameserver' \
+        /etc/resolv.conf | awk '{ print $2 '})
+
+    # prevent init scripts from running during install
+    echo $'#!/bin/sh\nexit 101' >$mountdir/usr/sbin/policy-rc.d
+    chmod a+x $mountdir/usr/sbin/policy-rc.d
+
+    chroot $mountdir /$(basename $cmd) $nameserver_ip
+
+    rm -rf $mountdir/usr/sbin/policy-rc.d
+
+    umount -f $mountdir/proc
+    umount -l $mountdir
+
+    qemu-img convert -c -o compat=0.10 -O qcow2 $raw_imgfile $imgfile
+
+    if dmsetup table | grep $loopdevice; then
+       dmsetup clear $loopdevice || true
+    fi
+}
+
+# cleanup (umount) the image
+cleanup() {
+    # designed to be idempotent
+    mount | grep $mountdir/proc && umount -l $mountdir/proc
+    mount | grep $mountdir && umount -l $mountdir
+    mount | grep "/mnt/${release}" && umount -l "/mnt/${release}"
+
+    if [ -f "${raw_imgfile}" ]; then
+        #kpartx -dv $raw_imgfile sometimes failed, we should checked it agein.
+        #if [ -z "$(kpartx -l $raw_imgfile | grep 'loop deleted')" ]; then
+        #    kpartx -dv $raw_imgfile
+        #fi
+        kpartx -dv $raw_imgfile || true
+    fi
+
+    rm -f $raw_imgfile
+    rm -rf $mountdir
+}
+
+exitcode=""
+error_trap()
+{
+    local rc=$?
+
+    set +e
+
+    if [ -z "$exitcode" ]; then
+        exitcode=$rc
+    fi
+
+    dmesg -T | tail -50
+
+    cleanup
+
+    echo "Image build failed with $exitcode"
+
+    exit $exitcode
+}
+
+main() {
+    cleanup
+
+    trap "error_trap" EXIT SIGTERM
+
+    download
+    setup
+    modify
+
+    trap - EXIT SIGTERM
+    cleanup
+
+    echo "the modified image is found here: $imgfile"
+}
+
+main
+
diff --git a/tools/ubuntu-server-cloudimg-dpdk-samplevnf-modify.sh b/tools/ubuntu-server-cloudimg-dpdk-samplevnf-modify.sh
new file mode 100755
index 00000000..924474d9
--- /dev/null
+++ b/tools/ubuntu-server-cloudimg-dpdk-samplevnf-modify.sh
@@ -0,0 +1,201 @@
+#!/bin/bash
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# installs required packages
+# must be run from inside the image (either chrooted or running)
+
+set -ex
+
+if [ $# -eq 1 ]; then
+    nameserver_ip=$1
+
+    # /etc/resolv.conf is a symbolic link to /run, restore at end
+    rm /etc/resolv.conf
+    echo "nameserver $nameserver_ip" > /etc/resolv.conf
+    echo "nameserver 8.8.8.8" >> /etc/resolv.conf
+    echo "nameserver 8.8.4.4" >> /etc/resolv.conf
+fi
+
+INSTALL_BIN_PATH="/opt/nsb_bin"
+BIN_PATH="/usr/sbin/"
+TREX_VERSION="v2.20"
+TREX_DOWNLOAD="https://trex-tgn.cisco.com/trex/release/$TREX_VERSION.tar.gz"
+TREX_DIR=$INSTALL_BIN_PATH/trex/scripts
+
+enable_proxy()
+{
+       echo $https_proxy
+       echo $http_proxy
+       if [[ "$http_proxy" != "" ]]; then
+           echo 'Acquire::http::Proxy "$http_proxy";' > /etc/apt/apt.conf
+       fi
+}
+
+install_trex()
+{
+
+	INSTALL_BIN_PATH="/opt/nsb_bin"
+	DPDK_DOWNLOAD="http://dpdk.org/browse/dpdk/snapshot/dpdk-16.07.zip"
+	TREX_VERSION="v2.20"
+	TREX_DOWNLOAD="https://trex-tgn.cisco.com/trex/release/$TREX_VERSION.tar.gz"
+	TREX_DIR=$INSTALL_BIN_PATH/trex/scripts
+        pushd .
+	cd $INSTALL_BIN_PATH
+	echo "Build TRex and installing Trex TG in $INSTALL_BIN_PATH/trex"
+	rm -rf ${TREX_DOWNLOAD##*/}
+	if [ ! -e ${TREX_DOWNLOAD##*/} ] ; then
+	    wget $TREX_DOWNLOAD
+	fi
+	tar zxvf ${TREX_DOWNLOAD##*/}
+	#pushd .
+	rm -rf $INSTALL_BIN_PATH/trex
+	mkdir -p $INSTALL_BIN_PATH/trex
+	mv $TREX_VERSION trex/scripts
+	rm -rf $TREX_VERSION.tar.gz
+	cd $INSTALL_BIN_PATH/trex/scripts/ko/src/
+	make
+	make install
+	touch "$INSTALL_BIN_PATH/trex/scripts/automation/trex_control_plane/stl/__init__.py"
+	cp "$INSTALL_BIN_PATH/trex/scripts/dpdk_nic_bind.py" "$INSTALL_BIN_PATH"
+        popd
+}
+install_sample_vnf()
+{
+	      mkdir -p $INSTALL_BIN_PATH
+        echo "Install Sample VNFs"
+        pushd .
+        cd $INSTALL_BIN_PATH
+        git clone https://git.opnfv.org/samplevnf
+        cd samplevnf
+        VNF_CORE=$INSTALL_BIN_PATH/samplevnf
+        ./tools/vnf_build.sh -s -d='17.02' -p='http://10.223.166.1:911'
+        cp $VNF_CORE/VNFs/vACL/build/vACL $INSTALL_BIN_PATH
+        cp $VNF_CORE/VNFs/vCGNAPT/build/vCGNAPT $INSTALL_BIN_PATH
+        cp $VNF_CORE/VNFs/vFW/build/vFW $INSTALL_BIN_PATH
+        cp $VNF_CORE/VNFs/DPPD-PROX/build/prox $INSTALL_BIN_PATH
+        cp $VNF_CORE/VNFs/UDP_Replay/build/UDP_Replay $INSTALL_BIN_PATH
+        cp $VNF_CORE/VNFs/vFW/vnf_template.txt $INSTALL_BIN_PATH
+
+				# copy to standard path
+        cp $VNF_CORE/VNFs/vACL/build/vACL $BIN_PATH
+        cp $VNF_CORE/VNFs/vCGNAPT/build/vCGNAPT $BIN_PATH
+        cp $VNF_CORE/VNFs/vFW/build/vFW $BIN_PATH
+        cp $VNF_CORE/VNFs/DPPD-PROX/build/prox $BIN_PATH
+        cp $VNF_CORE/VNFs/UDP_Replay/build/UDP_Replay $BIN_PATH
+
+        # build vpe
+        cd $VNF_CORE/dpdk/examples/ip_pipeline
+        export RTE_SDK= $VNF_CORE/dpdk
+        make
+        cp $VNF_CORE/build/ip_pipeline $INSTALL_BIN_PATH/vPE_vnf
+
+	      echo "Done"
+        popd
+}
+# iperf3 only available for trusty in backports
+if grep -q trusty /etc/apt/sources.list ; then
+    if [ "${YARD_IMG_ARCH}" = "arm64" ]; then
+        echo "deb [arch=${YARD_IMG_ARCH}] http://ports.ubuntu.com/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list
+    else
+        echo "deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted universe multiverse" >> /etc/apt/sources.list
+    fi
+fi
+
+# Workaround for building on CentOS (apt-get is not working with http sources)
+# sed -i 's/http/ftp/' /etc/apt/sources.list
+
+# Force apt to use ipv4 due to build problems on LF POD.
+echo 'Acquire::ForceIPv4 "true";' > /etc/apt/apt.conf.d/99force-ipv4
+
+echo 'GRUB_CMDLINE_LINUX="resume=/dev/sda1 default_hugepagesz=1G hugepagesz=1G hugepages=4 iommu=on iommu=pt intel_iommu=on"' >> /etc/default/grub
+echo 'huge /mnt/huge hugetlbfs defaults 0 0' >> vi /etc/fstab
+
+mkdir /mnt/huge
+chmod 777 /mnt/huge
+
+for i in {1..2}
+do
+    touch /etc/network/interfaces.d/eth$i.cfg
+    chmod 777 /etc/network/interfaces.d/eth$i.cfg
+    echo "auto eth$i" >> /etc/network/interfaces.d/eth$i.cfg
+    echo "iface eth$i inet dhcp" >> /etc/network/interfaces.d/eth$i.cfg
+done
+
+# this needs for checking dpdk status, adding interfaces to dpdk, bind, unbind etc..
+
+# Add hostname to /etc/hosts.
+# Allow console access via pwd
+cat <<EOF >/etc/cloud/cloud.cfg.d/10_etc_hosts.cfg
+manage_etc_hosts: True
+#password: RANDOM
+password: password
+chpasswd: { expire: False }
+ssh_pwauth: True
+EOF
+
+linuxheadersversion=$(echo ls boot/vmlinuz* | cut -d- -f2-)
+
+apt-get update
+apt-get install -y \
+    bc \
+    fio \
+    gcc \
+    git \
+    iperf3 \
+    iproute2 \
+    ethtool \
+    linux-tools-common \
+    linux-tools-generic \
+    lmbench \
+    make \
+    netperf \
+    patch \
+    perl \
+    rt-tests \
+    stress \
+    sysstat \
+    linux-headers-"${linuxheadersversion}" \
+    libpcap-dev \
+    chpasswd \
+    lua5.2
+
+# Build dpdk vPE VNF
+pushd .
+echo "root:root" | chpasswd
+enable_proxy
+install_sample_vnf
+install_trex
+popd .
+cd /root
+
+sed -i -e 's/PermitRootLogin without-password/PermitRootLogin yes/g'  /etc/ssh/sshd_config
+sed -i -e 's/prohibit-password/yes/g' /etc/ssh/sshd_config
+passwd
+
+CLONE_DEST=/opt/tempT
+# remove before cloning
+rm -rf -- "${CLONE_DEST}"
+git clone https://github.com/kdlucas/byte-unixbench.git "${CLONE_DEST}"
+make --directory "${CLONE_DEST}/UnixBench/"
+
+git clone https://github.com/beefyamoeba5/ramspeed.git "${CLONE_DEST}/RAMspeed"
+cd "${CLONE_DEST}/RAMspeed/ramspeed-2.6.0"
+mkdir temp
+bash build.sh
+
+git clone https://github.com/beefyamoeba5/cachestat.git "${CLONE_DEST}"/Cachestat
+
+# restore symlink
+ln -sfrT /run/resolvconf/resolv.conf /etc/resolv.conf