15 files changed, 95 insertions, 67 deletions
diff --git a/ansible/build_yardstick_image.yml b/ansible/build_yardstick_image.yml
index 429ab88aa..072c12c66 100644
--- a/ansible/build_yardstick_image.yml
+++ b/ansible/build_yardstick_image.yml
@@ -75,18 +75,18 @@
       ignore_errors: true
 
     - name: Debug dump loop devices
-      command: losetup
-      register: losetup_output
-
-    - debug:
-        var: losetup_output
-        verbosity: 2
+      command: losetup -a
+      ignore_errors: true
 
     - name: delete loop devices for image file
       # use this because kpartx -dv will fail if raw_imgfile was delete
       # but in theory we could have deleted file still attached to loopback device?
       # use grep because of // and awk
-      shell: losetup -O NAME,BACK-FILE | grep "{{ raw_imgfile_basename }}" | awk '{ print $1 }' | xargs -l1 losetup -d
+      shell: losetup -O NAME,BACK-FILE | grep "{{ raw_imgfile_basename }}" | awk '{ print $1 }' | xargs -l1 losetup -v -d
+      ignore_errors: true
+
+    - name: Debug dump loop devices again
+      command: losetup -a
       ignore_errors: true
 
     - name: delete {{ raw_imgfile }}
diff --git a/ansible/infra_deploy.yml b/ansible/infra_deploy.yml
index 4ad21af00..029879502 100644
--- a/ansible/infra_deploy.yml
+++ b/ansible/infra_deploy.yml
@@ -13,9 +13,12 @@
 # limitations under the License.
 ---
 - hosts: jumphost
+  vars:
+    rs_file: "{{ RS_FILE }}"
+    clean_up: "{{ CLEAN_UP | default(False) }}" # If True will be delete all VMs, networks, disk images
 
   roles:
-    - infra_check_requirements
     - infra_destroy_previous_configuration
+    - infra_check_requirements
     - infra_create_network
     - infra_create_vms
diff --git a/ansible/roles/docker/tasks/Debian.yml b/ansible/roles/docker/tasks/Debian.yml
index cf4128774..7f998de45 100644
--- a/ansible/roles/docker/tasks/Debian.yml
+++ b/ansible/roles/docker/tasks/Debian.yml
@@ -12,15 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
-  - name: add Ubuntu docker repo
-    apt_repository: repo='deb [trusted=yes] {{ ubuntu_docker_url }} ubuntu-{{ ansible_distribution_release }} main' state=present
-
-  - name: ensure correct docker version
-    action: "{{ ansible_pkg_mgr }} name={{ item }} state=present force=yes"
-    with_items: "{{ docker_packages[ansible_os_family] }}"
-
-  - name: remove Ubuntu docker repo
-    apt_repository:
-      repo: 'deb [trusted=yes] {{ ubuntu_docker_url }} ubuntu-{{ ansible_distribution_release }} main'
-      state: absent
-      update_cache: no
+  - name: Install docker.io
+    action: "{{ ansible_pkg_mgr }} name=docker.io state=present force=yes"
diff --git a/ansible/roles/docker/vars/main.yml b/ansible/roles/docker/vars/main.yml
index 8b5077490..a735d523d 100644
--- a/ansible/roles/docker/vars/main.yml
+++ b/ansible/roles/docker/vars/main.yml
@@ -16,5 +16,3 @@ docker_project_url: https://yum.dockerproject.org
 docker_packages:
   "RedHat":
     - docker-engine-1.13.1
-  "Debian":
-    - docker-engine=1.13.1*
diff --git a/ansible/roles/download_collectd/defaults/main.yml b/ansible/roles/download_collectd/defaults/main.yml
index 9beaeb897..3f5632394 100644
--- a/ansible/roles/download_collectd/defaults/main.yml
+++ b/ansible/roles/download_collectd/defaults/main.yml
@@ -1,4 +1,4 @@
 ---
 collectd_url: "https://github.com/collectd/collectd.git"
 collectd_dest: "{{ clone_dest }}/collectd"
-collectd_version: "c870991a2d614e51c03c0da76e9aef997343551b"
+collectd_version: "collectd-5.8"
diff --git a/ansible/roles/download_dpdk/defaults/main.yml b/ansible/roles/download_dpdk/defaults/main.yml
index fb42ed48a..d548280f5 100644
--- a/ansible/roles/download_dpdk/defaults/main.yml
+++ b/ansible/roles/download_dpdk/defaults/main.yml
@@ -1,10 +1,14 @@
 ---
 dpdk_version: "17.02"
-dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.zip"
+dpdk_url: "http://dpdk.org/browse/dpdk/snapshot/dpdk-{{ dpdk_version }}.tar.gz"
 dpdk_file: "{{ dpdk_url|basename }}"
-dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]zip$', '') }}"
+dpdk_unarchive: "{{ dpdk_file|regex_replace('[.]tar[.]gz$', '') }}"
 dpdk_dest: "{{ clone_dest }}/"
+#Note DPDK 17.08 17.11 and 18.02 are currently unsupported due to prox build issues
 dpdk_sha256s:
-  "17.02": "sha256:c675285d5eab8e7e8537a10bdae6f03c603caa80cb4bf5b055ddd482e3b7d67f"
-  "16.04": "sha256:65e587af439b21abf54c644f86ba5ce7ea65657b3272480dcd78b769d2aecac3"
-  "16.07": "sha256:46fee52f9b3ff54df869414c69f122aea8f744de5ed5f8fb275a22cc9cefe094"
+  "16.07": "sha256:d876e4b2a7101f28e7e345d3c88e66afe877d15f0159c19c5bc5bc26b7b7d788"
+  "17.02": "sha256:b07b546e910095174bdb6152bb0d7ce057cc4b79aaa74771aeee4e8a7219fb38"
+  "17.05": "sha256:763bfb7e1765efcc949e79d645dc9f1ebd16591431ba0db5ce22becd928dcd0a"
+  "17.08": "sha256:3a08addbff45c636538514e9a5838fb91ea557661a4c071e03a9a6987d46e5b6" #unsupported
+  "17.11": "sha256:77a727bb3834549985f291409c9a77a1e8be1c9329ce4c3eb19a22d1461022e4" #unsupported
+  "18.02": "sha256:f1210310fd5f01a3babe3a09d9b3e5a9db791c2ec6ecfbf94ade9f893a0632b8" #unsupported
diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml
index 314ee30af..5e616335a 100644
--- a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml
+++ b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_network.yml
@@ -14,18 +14,18 @@
 ---
 - name: Destroy old networks created by virt
   virt_net:
-    name: "{{ network_item.name }}"
+    name: "{{ network_item }}"
     command: destroy
-  when: network_item.name in virt_nets.list_nets
+  when: clean_up | bool or network_item in deploy_nets
 
-# Ignoring erros as network can be created without being defined.
+# Ignoring errors as network can be created without being defined.
 # This can happen if a user manually creates a network using the virsh command.
 # If the network is not defined the undefine code will throw an error.
 - name: Undefine old networks defined by virt
   virt_net:
-    name: "{{ network_item.name }}"
+    name: "{{ network_item }}"
     command: undefine
-  when: network_item.name in virt_nets.list_nets
+  when: clean_up | bool or network_item in deploy_nets
   ignore_errors: yes
 
 - name: Check if "ovs-vsctl" command is present
@@ -34,15 +34,20 @@
   ignore_errors: yes
 
 - name: Destroy OVS bridge if it exists
-  command: ovs-vsctl --if-exists -- del-br "{{ network_item.name }}"
-  when: ovs_vsctl_present.rc == 0
+  command: ovs-vsctl --if-exists -- del-br "{{ network_item }}"
+  when:
+    - ovs_vsctl_present.rc == 0
+    - clean_up | bool or network_item in deploy_nets
+  ignore_errors: yes
 
 - name: Check if linux bridge is present
-  stat: path="{{ '/sys/class/net/'+network_item.name+'/brif/' }}"
+  stat: path="{{ '/sys/class/net/' + network_item + '/brif/' }}"
   register: check_linux_bridge
 
 - name: Remove linux bridge if it exists
   shell: |
-    ifconfig "{{ network_item.name }}" down
-    brctl delbr "{{ network_item.name }}"
-  when: check_linux_bridge.stat.exists
+    ifconfig "{{ network_item }}" down
+    brctl delbr "{{ network_item }}"
+  when:
+    - check_linux_bridge.stat.exists
+    - clean_up | bool or network_item in deploy_nets
diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml
index 5e43ee81e..91e949344 100644
--- a/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml
+++ b/ansible/roles/infra_destroy_previous_configuration/tasks/delete_vm.yml
@@ -16,14 +16,14 @@
 - name: Destroy old VMs
   virt:
     command: destroy
-    name: "{{ node_item.hostname }}"
-  when: node_item.hostname in virt_vms.list_vms
+    name: "{{ vmhost_item }}"
+  when: clean_up | bool or vmhost_item in deploy_vms
   ignore_errors: yes
 
 # Ignore errors as VM can be running while undefined
 - name: Undefine old VMs
   virt:
     command: undefine
-    name: "{{ node_item.hostname }}"
-  when: node_item.hostname in virt_vms.list_vms
+    name: "{{ vmhost_item }}"
+  when: clean_up | bool or vmhost_item in deploy_vms
   ignore_errors: yes
diff --git a/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml b/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml
index e6c2c0229..6c4aa33cf 100644
--- a/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml
+++ b/ansible/roles/infra_destroy_previous_configuration/tasks/main.yml
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
-- name: Include
+- name: Include input file
   include_vars:
     file: "{{ rs_file }}"
     name: infra_deploy_vars
@@ -25,16 +25,40 @@
   virt: command=list_vms
   register: virt_vms
 
+- set_fact:
+    deploy_vms: "{{ deploy_vms | default([]) + [item.hostname] }}"
+  with_items: "{{ infra_deploy_vars.nodes }}"
+
+- name: Define old disk images to delete
+  shell: virsh domblklist {{ item }} | awk '/\/.*/ { print $2 }'
+  when: clean_up | bool or item in deploy_vms
+  with_items: "{{ virt_vms.list_vms }}"
+  register: virt_img
+
+- set_fact:
+    images: "{{ images | default([]) + item.stdout_lines }}"
+  when: item.stdout_lines is defined and item.stdout_lines | length > 0
+  with_items: "{{ virt_img.results }}"
+
 - name: Destroy old VMs
   include_tasks: delete_vm.yml
-  extra_vars: "{{ virt_vms }}"
   loop_control:
-    loop_var: node_item
-  with_items: "{{ infra_deploy_vars.nodes }}"
+    loop_var: vmhost_item
+  with_items: "{{ virt_vms.list_vms }}"
+
+- set_fact:
+    deploy_nets: "{{ deploy_nets | default([]) + [item.name] }}"
+  with_items: "{{ infra_deploy_vars.networks }}"
 
 - name: Delete old networks
   include_tasks: delete_network.yml
-  extra_vars: "{{ virt_nets }}"
   loop_control:
     loop_var: network_item
-  with_items: "{{ infra_deploy_vars.networks }}"
+  with_items: "{{ virt_nets.list_nets }}"
+
+- name: Delete old disk images
+  file:
+    path: "{{ item }}"
+    state: absent
+  when: images is defined and images | length > 0
+  with_items: "{{ images }}"
diff --git a/ansible/roles/install_civetweb/defaults/main.yml b/ansible/roles/install_civetweb/defaults/main.yml
index 2235fe172..ed5ab27f2 100644
--- a/ansible/roles/install_civetweb/defaults/main.yml
+++ b/ansible/roles/install_civetweb/defaults/main.yml
@@ -15,6 +15,7 @@
 civetweb_dest: "{{ clone_dest }}/civetweb"
 civetweb_build_dependencies:
   Debian:
+    - libjson-c-dev=0.11-4ubuntu2
     - libjson0
     - libjson0-dev
     - libssl-dev
diff --git a/ansible/roles/install_dpdk/tasks/main.yml b/ansible/roles/install_dpdk/tasks/main.yml
index e82ad8363..5bcfb50b1 100644
--- a/ansible/roles/install_dpdk/tasks/main.yml
+++ b/ansible/roles/install_dpdk/tasks/main.yml
@@ -116,7 +116,7 @@
 
 - name: copy dpdk-devbind.py to correct location
   copy:
-    src: "{{ dpdk_devbind[dpdk_version] }}"
+    src: "{{ dpdk_devbind_usertools if dpdk_version|float >= 17.02 else dpdk_devbind_tools }}"
     dest: "{{ INSTALL_BIN_PATH }}/dpdk-devbind.py"
     remote_src: yes
     force: yes
diff --git a/ansible/roles/install_dpdk/vars/main.yml b/ansible/roles/install_dpdk/vars/main.yml
index 45bcc339b..957f47e99 100644
--- a/ansible/roles/install_dpdk/vars/main.yml
+++ b/ansible/roles/install_dpdk/vars/main.yml
@@ -1,9 +1,10 @@
 ---
-dpdk_make_arch: x86_64-native-linuxapp-gcc
+dpdk_make_archs:
+ "amd64": "x86_64-native-linuxapp-gcc"
+ "arm64": "arm64-native-linuxapp-gcc"
+dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}"
 dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra"
 hugetable_mount: /mnt/huge
-dpdk_devbind:
-  "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py"
-  "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py"
-  "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py"
+dpdk_devbind_tools: "{{ dpdk_path }}/tools/dpdk-devbind.py"
+dpdk_devbind_usertools: "{{ dpdk_path }}/usertools/dpdk-devbind.py"
 dpdk_pmd_path: /usr/lib/dpdk-pmd/
diff --git a/ansible/roles/install_dpdk_shared/tasks/Debian.yml b/ansible/roles/install_dpdk_shared/tasks/Debian.yml
index c77e4f96a..641d8f9cd 100755
--- a/ansible/roles/install_dpdk_shared/tasks/Debian.yml
+++ b/ansible/roles/install_dpdk_shared/tasks/Debian.yml
@@ -16,6 +16,7 @@
   action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
   with_items:
     - libpcap-dev
+    - libnuma-dev
 
 - name: Install kernel headers
   action: "{{ ansible_pkg_mgr }} name=linux-headers-{{ dpdk_kernel }} state=present"
diff --git a/ansible/roles/install_dpdk_shared/tasks/main.yml b/ansible/roles/install_dpdk_shared/tasks/main.yml
index 65954be04..94b9215bd 100644
--- a/ansible/roles/install_dpdk_shared/tasks/main.yml
+++ b/ansible/roles/install_dpdk_shared/tasks/main.yml
@@ -42,6 +42,15 @@
   set_fact:
     RTE_KERNELDIR: "/lib/modules/{{ dpdk_kernel }}/build"
 
+# make clean must be run here as DPDK-shared is a copy of the DPDK directory
+# which has already run the make command
+# no T= target for clean
+- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }}
+  args:
+    chdir: "{{ dpdk_shared_path }}"
+  environment:
+    RTE_KERNELDIR: "{{ RTE_KERNELDIR }}"
+
 - command: make -j {{ ansible_processor_vcpus }} config  T={{ dpdk_make_arch }} O={{ dpdk_make_arch }}
   args:
     chdir: "{{ dpdk_shared_path }}"
@@ -85,13 +94,6 @@
     regexp: '^CONFIG_RTE_EAL_PMD_PATH=""'
     line: 'CONFIG_RTE_EAL_PMD_PATH="{{ dpdk_pmd_path }}"'
 
-  # no T= target for clean
-- command: make -j {{ ansible_processor_vcpus }} clean O={{ dpdk_make_arch }}
-  args:
-    chdir: "{{ dpdk_shared_path }}"
-  environment:
-    RTE_KERNELDIR: "{{ RTE_KERNELDIR }}"
-
 # TODO: disable ASLR
 
 - command: make -j {{ ansible_processor_vcpus }}
diff --git a/ansible/roles/install_dpdk_shared/vars/main.yml b/ansible/roles/install_dpdk_shared/vars/main.yml
index 45bcc339b..b663cedd2 100644
--- a/ansible/roles/install_dpdk_shared/vars/main.yml
+++ b/ansible/roles/install_dpdk_shared/vars/main.yml
@@ -1,9 +1,8 @@
 ---
-dpdk_make_arch: x86_64-native-linuxapp-gcc
+dpdk_make_archs:
+ "amd64": "x86_64-native-linuxapp-gcc"
+ "arm64": "arm64-native-linuxapp-gcc"
+dpdk_make_arch: "{{ dpdk_make_archs[YARD_IMG_ARCH] }}"
 dpdk_module_dir: "/lib/modules/{{ dpdk_kernel }}/extra"
 hugetable_mount: /mnt/huge
-dpdk_devbind:
-  "16.07": "{{ dpdk_path }}/tools/dpdk-devbind.py"
-  "17.02": "{{ dpdk_path }}/usertools/dpdk-devbind.py"
-  "17.04": "{{ dpdk_path }}/usertools/dpdk-devbind.py"
 dpdk_pmd_path: /usr/lib/dpdk-pmd/