Migrates Apex to Python

Removes all bash libraries and converts almost all of the code to a
mixture of Python and Ansible.  utils.sh and clean.sh still exist.
clean.sh will be migrated fully to clean.py in another patch.

The Apex Python package is now built into the opnfv-apex-common RPM.  To
install locally do 'pip3 install .'.  To deploy:

opnfv-deploy -d <file> -n <file> --image-dir /root/apex/.build  -v --debug

Non-python files (THT yaml, settings files, ansible playbooks) are all
installed into /usr/share/opnfv-apex/.  The RPM will copy settings files
into /etc/opnfv-apex/.

JIRA: APEX-317

Change-Id: I3232f0329bcd13bce5a28da6a8c9c84d0b048024
Signed-off-by: Tim Rozet <trozet@redhat.com>

4 files changed, 59 insertions, 92 deletions

diff --git a/build/Makefile b/build/Makefile
index 43eae37a..74e72657 100644
--- a/build/Makefile
+++ b/build/Makefile
@@ -79,7 +79,7 @@ $(RPMREL):
 	rpmbuild --clean -ba rpm_specs/opnfv-apex-release.spec $(RPM_DIR_ARGS) -D "_release $(shell echo $(RELEASE) | tr -d '_-')"
 
 $(BUILD_DIR)/opnfv-apex-common.tar.gz:
-	pushd ../ && git archive --format=tar.gz --prefix=opnfv-apex-common-$(RPMVERS)/ HEAD > $(BUILD_DIR)/opnfv-apex-common.tar.gz
+	pushd ../ && git archive --format=tar.gz --prefix=opnfv-apex-$(RPMVERS)/ HEAD > $(BUILD_DIR)/opnfv-apex-common.tar.gz
 
 .PHONY: common-rpm-check
 common-rpm-check: $(BUILD_DIR)/opnfv-apex-common.tar.gz
@@ -99,14 +99,7 @@ $(RPMCOM):
 
 .PHONY: python-tests
 python-tests:
-	# clean previous coverage data
-	rm -rf ../tests/.coverage
-	rm -rf ../tests/htmlcov
-	# run nose tests
-	cd ../tests && PYTHONPATH=../lib/python/ nosetests-3.4 . --with-coverage --cover-package apex --cover-package apex_python_utils --cover-html --cover-min-percentage 90
-
-	# generate reports
-	cd ../tests && coverage3 report --include '*lib/python/*' -m
+	tox -e py35
 
 #######################
 #  PYTHON PEP8 CHECK  #
@@ -114,8 +107,7 @@ python-tests:
 
 .PHONY: python-pep8-check
 python-pep8-check:
-	pep8 ../lib/python
-	pep8 ../tests
+	tox -e pep8
 
 #############
 #  YAMLLINT #
diff --git a/build/domain.xml b/build/domain.xml
new file mode 100644
index 00000000..57a67d87
--- /dev/null
+++ b/build/domain.xml
@@ -0,0 +1,34 @@
+<domain type='%(engine)s'>
+  <name>%(name)s</name>
+  <memory unit='KiB'>%(memory)s</memory>
+  <vcpu>%(cpus)s</vcpu>
+  <cpu mode='host-passthrough'/>
+  <os>
+    <type arch='%(arch)s'>hvm</type>
+    <boot dev='%(bootdev)s'/>
+    <bootmenu enable='no'/>
+    %(direct_boot)s
+    %(kernel_args)s
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+    <pae/>
+  </features>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>restart</on_crash>
+  <devices>
+    <controller type='scsi' model='virtio-scsi' index='0'/>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2' cache='unsafe'/>
+      <source file='%(imagefile)s'/>
+      <target dev='sda' bus='%(diskbus)s'/>
+    </disk>
+    %(network)s
+    %(bm_network)s
+    %(enable_serial_console)s
+    %(user_interface)s
+  </devices>
+</domain>
diff --git a/build/rpm_specs/opnfv-apex-common.spec b/build/rpm_specs/opnfv-apex-common.spec
index ccb100f3..42bc42f9 100644
--- a/build/rpm_specs/opnfv-apex-common.spec
+++ b/build/rpm_specs/opnfv-apex-common.spec
@@ -1,7 +1,9 @@
-Name:		opnfv-apex-common
+%global srcname opnfv-apex
+
+Name:		python3-%{srcname}
 Version:	5.0
 Release:	%{_release}
-Summary:	Scripts for OPNFV deployment using RDO Manager
+Summary:	Scripts for OPNFV deployment using Apex
 
 Group:		System Environment
 License:	Apache 2.0
@@ -13,23 +15,25 @@ BuildRequires:  python-docutils python34-devel
 Requires:       opnfv-apex-sdn opnfv-apex-undercloud openvswitch qemu-kvm bridge-utils libguestfs-tools libvirt-python
 Requires:       initscripts net-tools iputils iproute iptables python34 python34-yaml python34-jinja2 python3-ipmi python2-virtualbmc
 Requires:       ipxe-roms-qemu >= 20160127-1
+Requires:       libvirt-devel
 
 %description
-Scripts for OPNFV deployment using RDO Manager
+Scripts for OPNFV deployment using Apex
 https://wiki.opnfv.org/apex
 
 %prep
-%setup -q
+%autosetup -n %{srcname}-%{version}
 
 %build
 rst2html docs/release/installation/index.rst docs/release/installation/installation-instructions.html
 rst2html docs/release/release-notes/release-notes.rst docs/release/release-notes/release-notes.html
+%py3_build
 
 %global __python %{__python3}
 
 %install
 mkdir -p %{buildroot}%{_bindir}/
-install ci/deploy.sh %{buildroot}%{_bindir}/opnfv-deploy
+%py3_install
 install ci/clean.sh %{buildroot}%{_bindir}/opnfv-clean
 install ci/util.sh %{buildroot}%{_bindir}/opnfv-util
 
@@ -37,67 +41,10 @@ mkdir -p %{buildroot}%{_sysconfdir}/bash_completion.d/
 install build/bash_completion_apex %{buildroot}%{_sysconfdir}/bash_completion.d/apex
 
 mkdir -p %{buildroot}%{_sysconfdir}/opnfv-apex/
-install config/deploy/os-nosdn-nofeature-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-noha.yaml
-install config/deploy/os-nosdn-bar-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-bar-noha.yaml
-install config/deploy/os-nosdn-bar-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-bar-ha.yaml
-install config/deploy/os-nosdn-fdio-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-fdio-noha.yaml
-install config/deploy/os-nosdn-fdio-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-fdio-ha.yaml
-install config/deploy/os-nosdn-ovs_dpdk-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-ovs_dpdk-noha.yaml
-install config/deploy/os-nosdn-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-ha.yaml
-install config/deploy/os-nosdn-performance-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-performance-ha.yaml
-install config/deploy/os-nosdn-ovs_dpdk-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-ovs_dpdk-ha.yaml
-install config/deploy/os-nosdn-kvm-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-kvm-ha.yaml
-install config/deploy/os-nosdn-kvm-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-kvm-noha.yaml
-install config/deploy/os-nosdn-kvm_ovs_dpdk-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-kvm_ovs_dpdk-ha.yaml
-install config/deploy/os-nosdn-kvm_ovs_dpdk-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-nosdn-kvm_ovs_dpdk-noha.yaml
-install config/deploy/os-odl-bgpvpn-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-bgpvpn-ha.yaml
-install config/deploy/os-odl-bgpvpn-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-bgpvpn-noha.yaml
-install config/deploy/os-odl-sfc-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-sfc-ha.yaml
-install config/deploy/os-odl-sfc-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-sfc-noha.yaml
-install config/deploy/os-odl-fdio-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-fdio-noha.yaml
-install config/deploy/os-odl_netvirt-fdio-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl_netvirt-fdio-noha.yaml
-install config/deploy/os-odl-fdio-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-fdio-ha.yaml
-install config/deploy/os-odl-fdio-dvr-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-fdio-dvr-ha.yaml
-install config/deploy/os-odl-fdio-dvr-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-fdio-dvr-noha.yaml
-install config/deploy/os-odl-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-nofeature-ha.yaml
-install config/deploy/os-odl-nofeature-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-nofeature-noha.yaml
-install config/deploy/os-odl-ovs_dpdk-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-ovs_dpdk-ha.yaml
-install config/deploy/os-odl-ovs_dpdk-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-ovs_dpdk-noha.yaml
-install config/deploy/os-odl-gluon-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-gluon-noha.yaml
-install config/deploy/os-ovn-nofeature-noha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-ovn-nofeature-noha.yaml
-install config/deploy/os-onos-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-onos-nofeature-ha.yaml
-install config/deploy/os-onos-sfc-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-onos-sfc-ha.yaml
-install config/deploy/os-ocl-nofeature-ha.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/os-ocl-nofeature-ha.yaml
-install config/network/network_settings.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/network_settings.yaml
-install config/network/network_settings_v6.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/network_settings_v6.yaml
-install config/network/network_settings_vpp.yaml %{buildroot}%{_sysconfdir}/opnfv-apex/network_settings_vpp.yaml
-
-
-mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/python/apex
-install lib/common-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/configure-deps-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/parse-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/virtual-setup-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/undercloud-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/overcloud-deploy-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/post-install-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/utility-functions.sh %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/configure-vm %{buildroot}%{_var}/opt/opnfv/lib/
-install lib/python/apex_python_utils.py %{buildroot}%{_var}/opt/opnfv/lib/python/
-mkdir -p %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/__init__.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/deploy_settings.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/ip_utils.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/inventory.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/network_environment.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/network_settings.py %{buildroot}%{python3_sitelib}/apex/
-install lib/python/apex/clean.py %{buildroot}%{python3_sitelib}/apex/
-mkdir -p %{buildroot}%{python3_sitelib}/apex/common
-install lib/python/apex/common/__init__.py %{buildroot}%{python3_sitelib}/apex/common/
-install lib/python/apex/common/constants.py %{buildroot}%{python3_sitelib}/apex/common/
-install lib/python/apex/common/utils.py %{buildroot}%{python3_sitelib}/apex/common/
-mkdir -p %{buildroot}%{_var}/opt/opnfv/lib/installer/onos/
-install lib/installer/domain.xml %{buildroot}%{_var}/opt/opnfv/lib/installer/
+cp -f %{buildroot}%{_datadir}/opnfv-apex/config/deploy/* %{buildroot}%{_sysconfdir}/opnfv-apex/
+cp -f %{buildroot}%{_datadir}/opnfv-apex/config/network/* %{buildroot}%{_sysconfdir}/opnfv-apex/
+rm -f %{buildroot}%{_sysconfdir}/opnfv-apex/os-odl-csit-noha.yaml
+rm -f %{buildroot}%{_sysconfdir}/opnfv-apex/deploy_settings.yaml
 
 mkdir -p %{buildroot}%{_docdir}/opnfv/
 install LICENSE.rst %{buildroot}%{_docdir}/opnfv/
@@ -111,22 +58,13 @@ install config/network/network_settings_vpp.yaml %{buildroot}%{_docdir}/opnfv/ne
 install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/inventory.yaml.example
 
 %files
-%defattr(644, root, root, -)
+%{python3_sitelib}/apex/
+%{python3_sitelib}/apex-*.egg-info
+%defattr(644, root, root, 644)
 %attr(755,root,root) %{_bindir}/opnfv-deploy
 %attr(755,root,root) %{_bindir}/opnfv-clean
 %attr(755,root,root) %{_bindir}/opnfv-util
-%{_var}/opt/opnfv/lib/common-functions.sh
-%{_var}/opt/opnfv/lib/configure-deps-functions.sh
-%{_var}/opt/opnfv/lib/parse-functions.sh
-%{_var}/opt/opnfv/lib/virtual-setup-functions.sh
-%{_var}/opt/opnfv/lib/undercloud-functions.sh
-%{_var}/opt/opnfv/lib/overcloud-deploy-functions.sh
-%{_var}/opt/opnfv/lib/post-install-functions.sh
-%{_var}/opt/opnfv/lib/utility-functions.sh
-%attr(755,root,root) %{_var}/opt/opnfv/lib/configure-vm
-%{_var}/opt/opnfv/lib/python/
-%{python3_sitelib}/apex/
-%{_var}/opt/opnfv/lib/installer/domain.xml
+%{_datadir}/opnfv-apex/
 %{_sysconfdir}/bash_completion.d/apex
 %{_sysconfdir}/opnfv-apex/os-nosdn-nofeature-noha.yaml
 %{_sysconfdir}/opnfv-apex/os-nosdn-bar-noha.yaml
@@ -160,6 +98,7 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %{_sysconfdir}/opnfv-apex/os-onos-sfc-ha.yaml
 %{_sysconfdir}/opnfv-apex/os-ocl-nofeature-ha.yaml
 %{_sysconfdir}/opnfv-apex/network_settings.yaml
+%{_sysconfdir}/opnfv-apex/network_settings_vlans.yaml
 %{_sysconfdir}/opnfv-apex/network_settings_v6.yaml
 %{_sysconfdir}/opnfv-apex/network_settings_vpp.yaml
 %doc %{_docdir}/opnfv/LICENSE.rst
@@ -173,6 +112,8 @@ install config/inventory/pod_example_settings.yaml %{buildroot}%{_docdir}/opnfv/
 %doc %{_docdir}/opnfv/inventory.yaml.example
 
 %changelog
+* Mon Aug 14 2017 Tim Rozet <trozet@redhat.com> - 5.0-4
+- Updated for python refactoring
 * Mon May 08 2017 Dan Radez <dradez@redhat.com> - 5.0-3
 - adding configure-vm
 * Tue Apr 11 2017 Dan Radez <dradez@redhat.com> - 5.0-2
diff --git a/build/variables.sh b/build/variables.sh
index d37be0d0..6f8a6696 100644
--- a/build/variables.sh
+++ b/build/variables.sh
@@ -14,7 +14,7 @@ QUAGGA_RPMS_DIR=${BUILD_DIR}/quagga_build_dir
 CACHE_DIR="$(dirname ${BUILD_ROOT})/.cache"
 CACHE_HISTORY=".cache_history"
 PATCHES_DIR="${BUILD_ROOT}/patches"
-BUILD_UTILS="$(dirname ${BUILD_ROOT})/lib/python/build_utils.py"
+BUILD_UTILS="$(dirname ${BUILD_ROOT})/apex/build/build_utils.py"
 
 
 rdo_images_uri=${RDO_IMAGES_URI:-https://images.rdoproject.org/ocata/delorean/current-tripleo/stable/}
@@ -52,4 +52,4 @@ fdio_pkgs=(
 'http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/vpp-plugins-17.04.1-3~ge3b7ad7~b72.x86_64.rpm'
 )
 
-honeycomb_pkg='http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/honeycomb-1.17.04.1-2073.noarch.rpm'
\ No newline at end of file
+honeycomb_pkg='http://artifacts.opnfv.org/apex/danube/fdio_common_rpms/honeycomb-1.17.04.1-2073.noarch.rpm'