Upgrade the jumphost offline support

JIRA: -

1. Add support for ubuntu 16.04.3
2. Update the package building scripts for ubuntu 14.04.3
3. Remove some hard code

Change-Id: I96e587b6b3aa2c39e14e5b0ec7db82b460e1fb29
Signed-off-by: Yifei Xue <xueyifei@huawei.com>

3 files changed, 151 insertions, 49 deletions

diff --git a/deploy/prepare.sh b/deploy/prepare.sh
index 0927347c..e4cfb95b 100755
--- a/deploy/prepare.sh
+++ b/deploy/prepare.sh
@@ -26,6 +26,12 @@ function install_docker()
        stable"
     sudo apt-get update
     sudo apt-get install -y docker-ce
+    sleep 5
+    sudo cat << EOF > /etc/docker/daemon.json
+{
+  "storage-driver": "devicemapper"
+}
+EOF
 
     sudo service docker start
     sudo service docker restart
@@ -134,7 +140,7 @@ function _pre_env_setup()
 
      tar -zxvf $jhpkg_url -C $WORK_DIR/prepare/
      cd $WORK_DIR/prepare/jh_env_package
-     tar -zxvf trusty-jh-ppa.tar.gz
+     tar -zxvf jh-ppa.tar.gz
 
      if [[ ! -z /etc/apt/sources.list.d ]]; then
           mv /etc/apt/sources.list.d /etc/apt/sources.list.d.bak
@@ -144,7 +150,7 @@ function _pre_env_setup()
           mv /etc/apt/apt.conf /etc/apt/apt.conf.bak
      fi
 
-     cat << EOF > /etc/apt/apt.conf
+     sudo cat << EOF > /etc/apt/apt.conf
 APT::Get::Assume-Yes "true";
 APT::Get::force-yes "true";
 Acquire::http::Proxy::127.0.0.1:9998 DIRECT;
@@ -154,11 +160,17 @@ EOF
           mv /etc/apt/sources.list /etc/apt/sources.list.bak
      fi
 
-     cat << EOF > /etc/apt/sources.list
-deb [arch=amd64] http://127.0.0.1:9998/trusty-jh-ppa trusty main
+     sudo cat << EOF > /etc/apt/sources.list
+deb [arch=amd64] http://127.0.0.1:9998/jh-ppa $(lsb_release -cs) main
 EOF
 
-     nohup python -m SimpleHTTPServer 9998 &
+     if [[ $(lsb_release -cs) == "trusty" ]]; then
+         nohup python -m SimpleHTTPServer 9998 &
+     else
+         nohup python3 -m http.server 9998 &
+     fi
+
+     http_ppa_pid=$!
 
      cd -
      sleep 5
@@ -172,12 +184,25 @@ EOF
 
      sudo docker version >/dev/null 2>&1
      if [[ $? -ne 0 ]]; then
-         install_docker
+         sudo apt-get install -y docker-ce
+         sleep 5
+         sudo cat << EOF > /etc/docker/daemon.json
+{
+  "storage-driver": "devicemapper"
+}
+EOF
+
+         sudo service docker start
+         sudo service docker restart
+     else
+         StorageDriver=$(sudo docker info | grep "Storage Driver" | awk '{print $3}')
+         if [[ $StorageDriver != "devicemapper" ]]; then
+             echo "The storage driver of docker currently only supports 'devicemapper'."
+             exit 1
+         fi
      fi
 
-     pid=$(ps -ef | grep SimpleHTTPServer | grep 9998 | awk '{print $2}')
-     echo $pid
-     kill -9 $pid
+     kill -9 $http_ppa_pid
 
      if [[ ! -d /etc/libvirt/hooks ]]; then
          sudo mkdir -p /etc/libvirt/hooks
@@ -210,8 +235,6 @@ function _pre_pip_setup()
           mkdir -p ~/.pip
      fi
 
-#     rm -rf ~/.pip
-#     mkdir -p ~/.pip
      rm -rf $WORK_DIR/prepare
      mkdir -p $WORK_DIR/prepare
      jhpkg_url=${JHPKG_URL:7}
@@ -223,7 +246,7 @@ function _pre_pip_setup()
 
      tar -zxvf $jhpkg_url -C $WORK_DIR/prepare/
      cd $WORK_DIR/prepare/jh_env_package
-     tar -zxvf env_trusty_pip.tar.gz
+     tar -zxvf jh_pip.tar.gz
 
      cat << EOF > ~/.pip/pip.conf
 [global]
@@ -233,7 +256,14 @@ no-index = true
 trusted-host=127.0.0.1
 EOF
 
-     nohup python -m SimpleHTTPServer 9999 &
+     if [[ $(lsb_release -cs) == "trusty" ]]; then
+         nohup python -m SimpleHTTPServer 9999 &
+     else
+         nohup python3 -m http.server 9999 &
+     fi
+
+     http_pip_pid=$!
+     echo $http_pip_pid
 
      sleep 5
 
@@ -244,25 +274,27 @@ EOF
      virtualenv $WORK_DIR/venv
      source $WORK_DIR/venv/bin/activate
 
-     #pip install --upgrade cffi
-
-     PIP="cffi MarkupSafe pip cheetah pyyaml requests netaddr oslo.config ansible"
-
-     #PIP="paramiko jinja2 PyYAML setuptools pycrypto pyasn1 cryptography MarkupSafe idna six enum34 ipaddress pycparser virtualenv cheetah requests netaddr pbr oslo.config ansible"
-     for i in ${PIP}; do
-        pip install --upgrade $i
-     done
+     pip install cffi==1.10.0
+     pip install MarkupSafe==1.0
+     pip install pip==9.0.1
+     pip install cheetah==2.4.4
+     pip install pyyaml==3.12
+     pip install requests==2.18.1
+     pip install netaddr==0.7.19
+     pip install oslo.config==4.6.0
+     pip install ansible==2.3.1.0
+     sudo pip install docker-compose==1.14.0
+     if [[ $(lsb_release -cs) == "xenial" ]]; then
+         sudo pip install -U pyOpenSSL
+     fi
 
-     pid=$(ps -ef | grep SimpleHTTPServer | grep 9999 | awk '{print $2}')
-     echo $pid
-     kill -9 $pid
+     kill -9 $http_pip_pid
 
      if [[ -f ~/.pip/pip.conf.bak ]]; then
           mv ~/.pip/pip.conf.bak ~/.pip/pip.conf
      else
           rm -rf ~/.pip/pip.conf
      fi
-#     rm -rf ~/.pip/pip.conf
 }
 
 function prepare_python_env()
diff --git a/repo/jhenv_template/ubuntu/trusty/Dockerfile b/repo/jhenv_template/ubuntu/trusty/Dockerfile
index 4eeccb6c..99486bec 100644
--- a/repo/jhenv_template/ubuntu/trusty/Dockerfile
+++ b/repo/jhenv_template/ubuntu/trusty/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:14.04.3
+FROM ubuntu:14.04.5
 MAINTAINER Yifei Xue <xueyifei@huawei.com>
 
 ADD ./cp_env.sh /cp_env.sh
@@ -11,35 +11,72 @@ RUN apt-get install -y -d mkisofs bc curl ipmitool openvswitch-switch \
 git python-pip python-dev figlet \
 libxslt-dev libxml2-dev libvirt-dev \
 build-essential qemu-utils qemu-kvm libvirt-bin \
-virtinst libmysqld-dev \
+virtinst libmysqld-dev apparmor \
 libssl-dev libffi-dev python-cffi
 
-RUN wget -P /var/cache/apt/archives/ http://launchpadlibrarian.net/225838998/libffi6_3.1~rc1+r3.0.13-12ubuntu0.1_amd64.deb
-
 RUN mkdir jh_deb
 
 RUN cp /var/cache/apt/archives/*.deb jh_deb/
 
-RUN apt-get install -y reprepro
+#RUN wget -O jh_deb/gcc-5-base_5.4.0-6ubuntu1~16.04.5_amd64.deb http://launchpadlibrarian.net/336920226/gcc-5-base_5.4.0-6ubuntu1~16.04.5_amd64.deb
+
+#RUN wget -O jh_deb/libstdc++6_5.4.0-6ubuntu1~16.04.5_amd64.deb http://launchpadlibrarian.net/336920453/libstdc++6_5.4.0-6ubuntu1~16.04.5_amd64.deb
+
+#RUN wget -O jh_deb/libssl1.0.0_1.0.2g-1ubuntu4.9_amd64.deb http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.0.0_1.0.2g-1ubuntu4.9_amd64.deb
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-extra-4.4.0-31-generic
+
+RUN apt-get install -y reprepro curl \
+linux-image-extra-virtual apt-transport-https ca-certificates \
+software-properties-common
+
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+RUN apt-key fingerprint 0EBFCD88
+
+RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+trusty stable"
+
+RUN apt-get update
+
+RUN apt-get install -y docker-ce -d
+
+RUN cp /var/cache/apt/archives/*.deb jh_deb/
 
-RUN mkdir -p trusty-jh-ppa/conf
+RUN mkdir -p jh-ppa/conf
 
 RUN echo 'Codename: trusty\n\
 Components: main\n\
 Architectures: amd64\n'\
->> trusty-jh-ppa/conf/distributions
+>> jh-ppa/conf/distributions
 
-RUN reprepro -b trusty-jh-ppa includedeb trusty jh_deb/*.deb
+RUN reprepro -b jh-ppa includedeb trusty jh_deb/*.deb
 
-RUN tar -zcvf /jh-ppa-trusty.tar.gz ./trusty-jh-ppa
+RUN tar -zcvf /jh-ppa.tar.gz ./jh-ppa
 
-RUN apt-get install -y python-pip
+RUN apt-get install -y python-pip ansible libffi-dev python-dev
 
 RUN mkdir jh_pip
 
-RUN pip install --upgrade paramiko jinja2 PyYAML setuptools pycrypto pyasn1 \
-cryptography MarkupSafe idna six enum34 ipaddress cffi pycparser \
-virtualenv cheetah requests netaddr pbr oslo.config ansible -d jh_pip/
+RUN pip install virtualenv \
+cffi==1.10.0 \
+MarkupSafe==1.0 \
+pip==9.0.1 \
+cheetah==2.4.4 \
+pyyaml==3.12 \
+netaddr==0.7.19 \
+oslo.config==4.6.0 \
+cryptography==1.9 \
+ansible==2.3.1.0 \
+docker-compose==1.14.0 \
+-d jh_pip/
+
+RUN pip install pyOpenSSL \
+cryptography==1.9 \
+-d jh_pip/
+
+RUN pip install requests==2.18.1 \
+-d jh_pip/
 
 RUN tar -zcvf jh_pip.tar.gz jh_pip/
 
diff --git a/repo/jhenv_template/ubuntu/xenial/Dockerfile b/repo/jhenv_template/ubuntu/xenial/Dockerfile
index 6b9b517a..42b34c91 100644
--- a/repo/jhenv_template/ubuntu/xenial/Dockerfile
+++ b/repo/jhenv_template/ubuntu/xenial/Dockerfile
@@ -11,35 +11,68 @@ RUN apt-get install -y -d mkisofs bc curl ipmitool openvswitch-switch \
 git python-pip python-dev figlet \
 libxslt-dev libxml2-dev libvirt-dev \
 build-essential qemu-utils qemu-kvm libvirt-bin \
-virtinst libmysqld-dev \
+virtinst libmysqld-dev apparmor \
 libssl-dev libffi-dev python-cffi
 
-#RUN wget -P /var/cache/apt/archives/ http://launchpadlibrarian.net/225838998/libffi6_3.1~rc1+r3.0.13-12ubuntu0.1_amd64.deb
-
 RUN mkdir jh_deb
 
 RUN cp /var/cache/apt/archives/*.deb jh_deb/
 
-RUN apt-get install -y reprepro
+RUN wget -O jh_deb/gcc-5-base_5.4.0-6ubuntu1~16.04.5_amd64.deb http://launchpadlibrarian.net/336920226/gcc-5-base_5.4.0-6ubuntu1~16.04.5_amd64.deb
+
+RUN wget -O jh_deb/libstdc++6_5.4.0-6ubuntu1~16.04.5_amd64.deb http://launchpadlibrarian.net/336920453/libstdc++6_5.4.0-6ubuntu1~16.04.5_amd64.deb
+
+RUN wget -O jh_deb/libssl1.0.0_1.0.2g-1ubuntu4.9_amd64.deb http://security.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.0.0_1.0.2g-1ubuntu4.9_amd64.deb
+
+RUN apt-get install -y reprepro curl linux-image-extra-4.4.0-87-generic \
+linux-image-extra-virtual apt-transport-https ca-certificates \
+software-properties-common
+
+RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+
+RUN apt-key fingerprint 0EBFCD88
+
+RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+xenial stable"
+
+RUN apt-get update
+
+RUN apt-get install -y docker-ce -d
+
+RUN cp /var/cache/apt/archives/*.deb jh_deb/
 
-RUN mkdir -p xenial-jh-ppa/conf
+RUN mkdir -p jh-ppa/conf
 
 RUN echo 'Codename: xenial\n\
 Components: main\n\
 Architectures: amd64\n'\
->> xenial-jh-ppa/conf/distributions
+>> jh-ppa/conf/distributions
 
-RUN reprepro -b xenial-jh-ppa includedeb xenial jh_deb/*.deb
+RUN reprepro -b jh-ppa includedeb xenial jh_deb/*.deb
 
-RUN tar -zcvf /jh-ppa-xenial.tar.gz ./xenial-jh-ppa
+RUN tar -zcvf /jh-ppa.tar.gz ./jh-ppa
 
 RUN apt-get install -y python-pip
 
 RUN mkdir jh_pip
 
-RUN pip install --upgrade paramiko jinja2 PyYAML setuptools pycrypto pyasn1 \
-cryptography MarkupSafe idna six enum34 ipaddress cffi pycparser \
-virtualenv cheetah requests netaddr pbr oslo.config ansible -d jh_pip/
+RUN pip install virtualenv \
+cffi==1.10.0 \
+MarkupSafe==1.0 \
+pip==9.0.1 \
+cheetah==2.4.4 \
+pyyaml==3.12 \
+netaddr==0.7.19 \
+oslo.config==4.6.0 \
+ansible==2.3.1.0 \
+docker-compose==1.14.0 \
+-d jh_pip/
+
+RUN pip install pyOpenSSL \
+-d jh_pip/
+
+RUN pip install requests==2.18.1 \
+-d jh_pip/
 
 RUN tar -zcvf jh_pip.tar.gz jh_pip/