Merge "Prepare the jump host offline"

8 files changed, 253 insertions, 41 deletions

diff --git a/build.sh b/build.sh
index 5ec8973b..0bb6f17a 100755
--- a/build.sh
+++ b/build.sh
@@ -81,7 +81,7 @@ function download_packages()
      for i in $CENTOS_BASE $COMPASS_CORE $COMPASS_WEB $COMPASS_INSTALL $TRUSTY_JUNO_PPA $TRUSTY_LIBERTY_PPA $TRUSTY_MITAKA_PPA \
               $XENIAL_MITAKA_PPA $UBUNTU_ISO $UBUNTU_ISO1 $CENTOS_ISO $CENTOS7_JUNO_PPA $CENTOS7_KILO_PPA $CENTOS7_LIBERTY_PPA \
               $CENTOS7_MITAKA_PPA $LOADERS $CIRROS $APP_PACKAGE $COMPASS_PKG \
-              $PIP_REPO $ANSIBLE_MODULE; do
+              $PIP_REPO $ANSIBLE_MODULE $JH_ENV_PACKAGE; do
 
          if [[ ! $i ]]; then
              continue
@@ -169,6 +169,8 @@ function copy_file()
 
     tar -zxvf $CACHE_DIR/`basename $PIP_REPO` -C $new/
 
+    tar -zxvf $CACHE_DIR/`basename $JH_ENV_PACKAGE` -C $new/
+
     find $new/compass -name ".git" | xargs rm -rf
 }
 
diff --git a/build/build.conf b/build/build.conf
index c6e0f7a1..0723064f 100644
--- a/build/build.conf
+++ b/build/build.conf
@@ -14,6 +14,7 @@ export COMPASS_INSTALL=${COMPASS_INSTALL:-http://github.com/baigk/compass-instal
 # Compass core packages
 export COMPASS_PKG=${COMPASS_PKG:-$PACKAGE_URL/centos7-compass-core.tar.gz}
 export PIP_REPO=${PIP_REPO:-$PACKAGE_URL/pip.tar.gz}
+export JH_ENV_PACKAGE=${JH_ENV_PACKAGE:-$PACKAGE_URL/jh_env_package.tar.gz}
 export ANSIBLE_MODULE=${ANSIBLE_MODULE:-https://github.com/openstack-ansible/openstack-ansible-modules.git}
 
 # OS ISO for provisioning
diff --git a/build/jhenv_template/centos/rhel7/Dockerfile b/build/jhenv_template/centos/rhel7/Dockerfile
new file mode 100644
index 00000000..a47360aa
--- /dev/null
+++ b/build/jhenv_template/centos/rhel7/Dockerfile
@@ -0,0 +1,54 @@
+FROM centos:7.1.1503
+MAINTAINER Yifei Xue <xueyifei@huawei.com>
+
+ADD ./cp_env.sh /cp_env.sh
+
+#RUN yum provides '*/applydeltarpm' -y
+
+#RUN yum install deltarpm -y
+
+RUN yum remove -y systemd
+
+RUN yum update -y
+
+RUN yum install yum-plugin-downloadonly
+
+RUN mkdir -p /rpm
+
+RUN yum install -y epel-release-7-8
+
+RUN yum groupinstall -y --downloadonly --downloaddir=/rpm 'Development Tools'
+
+RUN yum install -y --downloadonly --downloaddir=/rpm epel-release mkisofs bc curl ipmitool \
+git python-pip python-devel figlet \
+libxslt-devel libxml2-devel libvirt-devel \
+qemu qemu-kvm qemu-img virt-manager libvirt libvirt-python libvirt-client virt-install virt-viewer \
+guestfish libguestfs-tools virt-top mysql-devel \
+openssl-devel libffi libffi-devel python-cffi
+
+RUN yum install -y epel-release mkisofs bc curl ipmitool \
+git python-pip python-devel figlet \
+libxslt-devel libxml2-devel libvirt-devel \
+qemu qemu-kvm qemu-img virt-manager libvirt libvirt-python libvirt-client virt-install virt-viewer \
+guestfish libguestfs-tools virt-top mysql-devel \
+openssl-devel libffi libffi-devel python-cffi
+
+RUN yum install -y python-pip
+
+RUN yum install -y tar
+
+RUN tar -zcvf env_rel7_rpm.tar.gz /rpm
+
+RUN mkdir -p /home/pip
+
+RUN pip install --upgrade markupsafe virtualenv netaddr pbr oslo.config Cheetah ansible -d /home/pip/
+
+RUN tar -zcvf env_rel7_pip.tar.gz /home/pip
+
+RUN mkdir -p /home/tmp
+
+RUN cp *.tar.gz /home/tmp
+
+RUN chmod +x /cp_env.sh
+
+CMD ["/cp_env.sh"]
diff --git a/build/jhenv_template/ubuntu/trusty/Dockerfile b/build/jhenv_template/ubuntu/trusty/Dockerfile
new file mode 100644
index 00000000..650d763c
--- /dev/null
+++ b/build/jhenv_template/ubuntu/trusty/Dockerfile
@@ -0,0 +1,35 @@
+FROM ubuntu:14.04.3
+MAINTAINER Yifei Xue <xueyifei@huawei.com>
+
+ADD ./cp_env.sh /cp_env.sh
+
+RUN apt-get update
+
+RUN apt-get install -y -d mkisofs bc curl ipmitool openvswitch-switch \
+git python-pip python-dev figlet \
+libxslt-dev libxml2-dev libvirt-dev \
+build-essential qemu-utils qemu-kvm libvirt-bin \
+virtinst libmysqld-dev \
+libssl-dev libffi-dev python-cffi
+
+RUN mkdir jh_deb
+
+RUN cp /var/cache/apt/archives/*.deb jh_deb/
+
+RUN tar -zcvf env_trusty_deb.tar.gz jh_deb/
+
+RUN apt-get install -y python-pip
+
+RUN mkdir jh_pip
+
+RUN pip install --upgrade markupsafe virtualenv cheetah pyyaml requests netaddr pbr oslo.config ansible -d jh_pip/
+
+RUN tar -zcvf env_trusty_pip.tar.gz jh_pip/
+
+RUN mkdir -p /home/tmp
+
+RUN cp *.tar.gz /home/tmp
+
+RUN chmod +x /cp_env.sh
+
+CMD ["/cp_env.sh"]
diff --git a/build/jhenv_template/ubuntu/xenial/Dockerfile b/build/jhenv_template/ubuntu/xenial/Dockerfile
new file mode 100644
index 00000000..f70586ea
--- /dev/null
+++ b/build/jhenv_template/ubuntu/xenial/Dockerfile
@@ -0,0 +1,35 @@
+FROM ubuntu:16.04
+MAINTAINER Yifei Xue <xueyifei@huawei.com>
+
+ADD ./cp_env.sh /cp_env.sh
+
+RUN apt-get update
+
+RUN apt-get install -y -d mkisofs bc curl ipmitool openvswitch-switch \
+git python-pip python-dev figlet \
+libxslt-dev libxml2-dev libvirt-dev \
+build-essential qemu-utils qemu-kvm libvirt-bin \
+virtinst libmysqld-dev \
+libssl-dev libffi-dev python-cffi
+
+RUN mkdir jh_deb
+
+RUN cp /var/cache/apt/archives/*.deb jh_deb/
+
+RUN tar -zcvf env_xenial_deb.tar.gz jh_deb/
+
+RUN apt-get install -y python-pip
+
+RUN mkdir jh_pip
+
+RUN pip install --upgrade markupsafe virtualenv cheetah pyyaml requests netaddr pbr oslo.config ansible -d jh_pip/
+
+RUN tar -zcvf env_xenial_pip.tar.gz jh_pip/
+
+RUN mkdir -p /home/tmp
+
+RUN cp *.tar.gz /home/tmp
+
+RUN chmod +x /cp_env.sh
+
+CMD ["/cp_env.sh"]
diff --git a/build/make_repo.sh b/build/make_repo.sh
index f38da7ff..f186c46c 100755
--- a/build/make_repo.sh
+++ b/build/make_repo.sh
@@ -33,6 +33,7 @@ set -ex
 cp /*.tar.gz /result -f
 EOF
 
+    sudo apt-get -f install
     sudo apt-get install python-yaml -y
     sudo apt-get install python-cheetah -y
 }
@@ -42,7 +43,7 @@ function make_repo()
     rm -f ${WORK_PATH}/work/repo/install_packages.sh
     rm -f ${WORK_PATH}/work/repo/Dockerfile
 
-    option=`echo "os-ver:,package-tag:,tmpl:,default-package:, \
+    option=`echo "os-ver:,jh-os:,package-tag:,tmpl:,default-package:, \
             special-package:,special-package-script-dir:, \
             special-package-dir:,ansible-dir:,special-package-dir" | sed 's/ //g'`
 
@@ -53,6 +54,7 @@ function make_repo()
     eval set -- "$TEMP"
 
     os_ver=""
+    jh_os=""
     package_tag=""
     tmpl=""
     default_package=""
@@ -64,6 +66,7 @@ function make_repo()
     while :; do
         case "$1" in
             --os-ver) os_ver=$2; shift 2;;
+            --jh-os) jh_os=$2; shift 2;;
             --package-tag) package_tag=$2; shift 2;;
             --tmpl) tmpl=$2; shift 2;;
             --default-package) default_package=$2; shift 2;;
@@ -81,6 +84,11 @@ function make_repo()
         return
     fi
 
+    if [[ -n ${package_tag} && ${package_tag} == "jhenv" && -n ${jh_os} ]]; then
+        make_jhenv_repo
+        return
+    fi
+
     if [[ -z ${os_ver} || -z ${package_tag} ]]; then
         echo "parameter is wrong"
         exit 1
@@ -267,8 +275,53 @@ function make_pip_repo()
     cd $WORK_PATH/work/repo; tar -zcvf pip.tar.gz ./pip; cd -
 }
 
+function make_jhenv_repo()
+{
+    if [[ ${jh_os} == trusty ]]; then
+        env_os_name=ubuntu
+    fi
+
+    if [[ ${jh_os} == xenial ]]; then
+        env_os_name=ubuntu
+    fi
+
+    if [[ ${jh_os} =~ rhel[0-9]*$ ]]; then
+        env_os_name=centos
+    fi
+
+    if [[ -d ${WORK_PATH}/build/jhenv_template/$env_os_name ]]; then
+
+        jh_env_dockerfile=Dockerfile
+        jh_env_docker_tmpl=${BUILD_PATH}/jhenv_template/$env_os_name/$jh_os/${jh_env_dockerfile}".tmpl"
+        jh_env_docker_tag="$jh_os/env"
+
+        rm -rf ${WORK_PATH}/work/repo/jhenv_template
+        mkdir ${WORK_PATH}/work/repo/jhenv_template
+        cp -rf ${WORK_PATH}/build/jhenv_template/$env_os_name/$jh_os/${jh_env_dockerfile} ${WORK_PATH}/work/repo/jhenv_template
+
+cat <<EOF >${WORK_PATH}/work/repo/jhenv_template/cp_env.sh
+#!/bin/bash
+set -ex
+cp /*.tar.gz /env -f
+EOF
+
+        sudo docker build --no-cache=true -t ${jh_env_docker_tag} -f ${WORK_PATH}/work/repo/jhenv_template/${jh_env_dockerfile} ${WORK_PATH}/work/repo/jhenv_template
+        sudo docker run -t -v ${WORK_PATH}/work/repo:/env ${jh_env_docker_tag}
+
+        image_id=$(sudo docker images|grep ${jh_env_docker_tag}|awk '{print $3}')
+
+        sudo docker rmi -f ${image_id}
+
+#    cd $WORK_PATH/work/repo; tar -zcvf pip.tar.gz ./pip; cd -
+    fi
+}
+
 function make_all_repo()
 {
+    for env_os in trusty xanial rhel7; do
+    make_repo --package-tag jhenv --jh-os $env_os
+    done
+
     make_repo --package-tag pip
 
     make_repo --os-ver rhel7 --package-tag compass \
diff --git a/deploy/launch.sh b/deploy/launch.sh
index 488e0fd7..2c272c79 100755
--- a/deploy/launch.sh
+++ b/deploy/launch.sh
@@ -15,7 +15,6 @@ mkdir -p $WORK_DIR/script
 export DEPLOY_FIRST_TIME=${DEPLOY_FIRST_TIME-"true"}
 
 source ${COMPASS_DIR}/deploy/prepare.sh
-prepare_python_env
 source ${COMPASS_DIR}/util/log.sh
 source ${COMPASS_DIR}/deploy/deploy_parameter.sh
 source $(process_input_para $*) || exit 1
@@ -30,6 +29,16 @@ source ${COMPASS_DIR}/deploy/compass_vm.sh
 source ${COMPASS_DIR}/deploy/deploy_host.sh
 
 ######################### main process
+download_iso
+pre_prepare
+prepare_env
+if [[ "$DEPLOY_COMPASS" == "true" ]]; then
+    if ! prepare_python_env;then
+        echo "prepare_python_env failed"
+        exit 1
+    fi
+fi
+
 print_logo
 
 if [[ ! -z $VIRT_NUMBER ]];then
@@ -46,11 +55,6 @@ fi
 export machines
 
 if [[ "$DEPLOY_COMPASS" == "true" ]]; then
-    if ! prepare_env;then
-        echo "prepare_env failed"
-        exit 1
-    fi
-
     log_info "########## set up network begin #############"
     if ! create_nets;then
         log_error "create_nets failed"
diff --git a/deploy/prepare.sh b/deploy/prepare.sh
index a4f606af..7d7a02d5 100755
--- a/deploy/prepare.sh
+++ b/deploy/prepare.sh
@@ -9,11 +9,11 @@
 ##############################################################################
 function print_logo()
 {
-    if ! apt --installed list 2>/dev/null | grep "figlet"
-    then
-        sudo apt-get update -y
-        sudo apt-get install -y --force-yes figlet
-    fi
+#    if ! apt --installed list 2>/dev/null | grep "figlet"
+#    then
+#        sudo apt-get update -y
+#        sudo apt-get install -y --force-yes figlet
+#    fi
 
     figlet -ctf slant Compass Installer
     set +x; sleep 2; set -x
@@ -32,32 +32,19 @@ function download_iso()
         fi
     fi
 
+    rm -rf $WORK_DIR/iso
+    mkdir -p $WORK_DIR/iso
+    mkdir -p $WORK_DIR/cache
     curl --connect-timeout 10 -o $WORK_DIR/cache/$iso_name $ISO_URL
-}
 
-function prepare_env() {
-   if [[ "$DEPLOY_FIRST_TIME" == "true" ]]; then
-        sudo apt-get update -y
-        sudo apt-get install -y --force-yes mkisofs bc curl ipmitool openvswitch-switch
-        sudo apt-get install -y --force-yes git python-dev
-        sudo apt-get install -y --force-yes libxslt-dev libxml2-dev libvirt-dev build-essential qemu-utils qemu-kvm libvirt-bin virtinst libmysqld-dev
-        sudo apt-get install -y --force-yes libffi-dev libssl-dev
-    fi
-
-    sudo service libvirt-bin restart
-    if sudo service openvswitch-switch status|grep stop; then
-        sudo service openvswitch-switch start
-    fi
+}
 
+function pre_prepare() {
     # prepare work dir
-    rm -rf $WORK_DIR/{installer,vm,network,iso}
+    rm -rf $WORK_DIR/{installer,vm,network}
     mkdir -p $WORK_DIR/installer
     mkdir -p $WORK_DIR/vm
     mkdir -p $WORK_DIR/network
-    mkdir -p $WORK_DIR/iso
-    mkdir -p $WORK_DIR/cache
-
-    download_iso
 
     cp $WORK_DIR/cache/`basename $ISO_URL` $WORK_DIR/iso/centos.iso -f
 
@@ -72,24 +59,65 @@ function prepare_env() {
     chmod 755 $WORK_DIR -R
 
     sudo cp ${COMPASS_DIR}/deploy/qemu_hook.sh /etc/libvirt/hooks/qemu
+
+    rm -rf $WORK_DIR/cache/compass_tmp
+    mkdir -p $WORK_DIR/cache/compass_tmp
+    rm -rf $WORK_DIR/cache/jh_env_tmp
+    mkdir -p $WORK_DIR/cache/jh_env_tmp
+    sudo mount -o loop $WORK_DIR/cache/$iso_name $WORK_DIR/cache/compass_tmp/
+    cp $WORK_DIR/cache/compass_tmp/jh_env_package/*.tar.gz $WORK_DIR/cache/jh_env_tmp/
+    sudo umount $WORK_DIR/cache/compass_tmp/
+    rm -rf $WORK_DIR/cache/compass_tmp
+    tar -zxvf $WORK_DIR/cache/jh_env_tmp/env_trusty_deb.tar.gz -C $WORK_DIR/cache/jh_env_tmp
+    tar -zxvf $WORK_DIR/cache/jh_env_tmp/env_trusty_pip.tar.gz -C $WORK_DIR/cache/jh_env_tmp
+}
+
+function prepare_env() {
+   if [[ "$DEPLOY_FIRST_TIME" == "true" ]]; then
+        cd $WORK_DIR/cache/jh_env_tmp/jh_deb
+        dpkg -i *.deb
+        cd -
+   fi
 }
 
 function  _prepare_python_env() {
    rm -rf $WORK_DIR/venv
    mkdir -p $WORK_DIR/venv
 
-   sudo apt-get install -y --force-yes python-pip
-   sudo pip install --upgrade virtualenv
+   rm -rf ~/.pip
+   mkdir -p ~/.pip
+   cd $WORK_DIR/cache/jh_env_tmp/
+
+cat <<EOF > ~/.pip/pip.conf
+[global]
+find-links = http://127.0.0.1:9999/jh_pip
+no-index = true
+[install]
+trusted-host=127.0.0.1
+EOF
+
+   nohup python -m SimpleHTTPServer 9999 &
+
+   cd -
+
    virtualenv $WORK_DIR/venv
    source $WORK_DIR/venv/bin/activate
 
-   pip install --upgrade pip
-   pip install --upgrade cheetah
-   pip install --upgrade pyyaml
-   pip install --upgrade requests
-   pip install --upgrade netaddr
-   pip install --upgrade oslo.config
-   pip install --upgrade ansible
+   PIP="markupsafe virtualenv cheetah pyyaml requests netaddr pbr oslo.config ansible"
+   for i in ${PIP}; do
+     pip install --upgrade $i
+   done
+   service libvirt-bin restart
+   if sudo service openvswitch-switch status|grep stop; then
+       sudo service openvswitch-switch start
+   fi
+
+   pid=$(ps -ef | grep SimpleHTTPServer | grep 9999 | awk '{print $2}')
+   echo $pid
+   kill -9 $pid
+
+   rm -rf ~/.pip/pip.conf
+   rm -rf $WORK_DIR/cache/jh_env_tmp
 }
 
 function prepare_python_env()