Improve VMs spawning for OpenStack

In some, like 5% or so, VMs become not responsible. Added a validation to make sure VMs
are reachable over SSH. In case of issue - restarting VM.

JIRA: YARDSTICK-1141

Change-Id: I110fe35b4b7bdbc4ba962d3814ce0f8cd40e5400
Signed-off-by: Stepan Andrushko <stepanx.andrushko@intel.com>

3 files changed, 65 insertions, 3 deletions

diff --git a/ansible/roles/infra_create_vms/tasks/main.yml b/ansible/roles/infra_create_vms/tasks/main.yml
index 4d47f44ff..b422a9205 100644
--- a/ansible/roles/infra_create_vms/tasks/main.yml
+++ b/ansible/roles/infra_create_vms/tasks/main.yml
@@ -32,3 +32,14 @@
   loop_control:
     loop_var: node_item
   with_items: "{{ infra_deploy_vars.nodes }}"
+
+- name: Create list of dictionaries with vm name, ip address
+  set_fact:
+    vm_name_ip: "{{ vm_name_ip|default([]) + [{item.hostname: item.interfaces[1].ip}] }}"
+  with_items: "{{ infra_deploy_vars.nodes }}"
+
+- name: Make sure VM is reachable
+  include_tasks: validate_vms.yml
+  loop_control:
+    loop_var: name_ip
+  with_items: "{{ vm_name_ip }}"
diff --git a/ansible/roles/infra_create_vms/tasks/validate_vms.yml b/ansible/roles/infra_create_vms/tasks/validate_vms.yml
new file mode 100644
index 000000000..ce5eff211
--- /dev/null
+++ b/ansible/roles/infra_create_vms/tasks/validate_vms.yml
@@ -0,0 +1,54 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Validate if VM is responding
+  wait_for:
+    host: "{{ item.value }}"
+    port: 22
+    timeout: 10
+  with_dict: "{{ name_ip }}"
+  register: result
+  ignore_errors: yes
+
+- name: Stop VM if it is not responding
+  virt:
+    name: "{{ item.key }}"
+    command: destroy
+  when: result is failed
+  with_dict: "{{ name_ip }}"
+
+- name: Wait for VM is being stopped
+  pause:
+    seconds: 10
+  when: result is failed
+
+- name: Start VM
+  virt:
+    name: "{{ item.key }}"
+    command: start
+  when: result is failed
+  with_dict: "{{ name_ip }}"
+
+- name: Validate if VM is responding
+  wait_for:
+    host: "{{ item.value }}"
+    port: 22
+    timeout: 10
+  when: result is failed
+  with_dict: "{{ name_ip }}"
+  register: result
+
+- fail:
+    msg: "FAILED on {{ name_ip }}"
+  when: result is failed
diff --git a/ansible/roles/infra_prepare_vms/tasks/main.yml b/ansible/roles/infra_prepare_vms/tasks/main.yml
index d7ed08511..4e0d9c373 100644
--- a/ansible/roles/infra_prepare_vms/tasks/main.yml
+++ b/ansible/roles/infra_prepare_vms/tasks/main.yml
@@ -100,6 +100,3 @@
     secondary_ip: "{{ item.interfaces[1].ip }}"
   when: item.hostname == 'yardstickvm'
   with_items: "{{ infra_deploy_vars.nodes }}"
-
-- name: Workaround, not all VMs are ready by that time
-  pause: seconds=20