xci: roles: create-vm-nodes: Add role for creating XCI VM nodes

Add a new role based on the bifrost one to create nodes for the bifrost
virtual deployments. This role will install and configure libvirt on the
host, download a prebuilt OPNFV VM image and deploy the OPNFV VM using
that image. Moreover, it will create the rest of the nodes for the
virtual deployment which will be configured by bifrost later on.

Change-Id: I9fbd084261351d3b53ae373060f43df046191c5e
Co-Authored-by: Markos Chandras <mchandras@suse.de>
Signed-off-by: Manuel Buil <mbuil@suse.com>

4 files changed, 366 insertions, 0 deletions

diff --git a/xci/playbooks/roles/create-vm-nodes/tasks/create_vm.yml b/xci/playbooks/roles/create-vm-nodes/tasks/create_vm.yml
new file mode 100644
index 00000000..d8169c2f
--- /dev/null
+++ b/xci/playbooks/roles/create-vm-nodes/tasks/create_vm.yml
@@ -0,0 +1,166 @@
+---
+# Create a VM and volume for it, save its MAC address
+- shell: "sudo virsh list --all | grep 'shut off' | wc -l"
+  register: num_vms
+
+- name: "Creating VM"
+  block:
+  # NOTE(pas-ha) item here refers to name of the vm
+  - set_fact:
+      vm_name: "{{ node_names[num_vms.stdout | int] }}"
+
+  - set_fact:
+      vm_log_file: "{{ node_logdir }}/{{ vm_name }}_console.log"
+      vm_host_group: "{{ vm_default_groups }}"
+
+  - set_fact:
+      vm_host_group: "{{ vm_default_groups | union(vm_groups[vm_name]) }}"
+    when: vm_groups[vm_name] is defined
+
+  - name: set prealloc arg for Debian
+    set_fact:
+      prealloc: "--prealloc-metadata"
+    when:
+      - ansible_os_family == 'Debian'
+      - vm_libvirt_uri == 'qemu:///system'
+
+  - name: list info on pools
+    virt_pool:
+      command: facts
+      uri: "{{ vm_libvirt_uri }}"
+
+  - name: list existing vms
+    virt:
+      command: list_vms
+    register: existing_vms
+
+  - block:
+    - name: Check if volume exists
+      stat:
+        path: "{{ opnfv_image_path }}/{{ vm_name }}.qcow2"
+      register: _vm_volume_prepared
+
+    # NOTE(pas-ha) Ansible still lacks modules to operate on libvirt volumes
+    # mbuil: Assuming there is only one disk [0]
+    - name: create volume for vm
+      command: >
+        virsh --connect {{ vm_libvirt_uri }}
+        vol-create-as {{ node_storage_pool }} {{ vm_name }}.qcow2
+        {{ item.disks[0].disk_capacity }}
+        --format qcow2 {{ prealloc|default("") }}
+      when:
+        - not _vm_volume_prepared.stat.exists
+        - (vm_name + '.qcow2') not in ansible_libvirt_pools[node_storage_pool].volumes
+
+    - name: set path to the volume created
+      set_fact:
+        vm_volume_path: "{{ ansible_libvirt_pools[node_storage_pool].path }}/{{ vm_name }}.qcow2"
+
+    - name: pre-touch the vm volume
+      file:
+        state: touch
+        path: "{{ vm_volume_path }}"
+      when: vm_libvirt_uri == 'qemu:///system'
+
+    # NOTE(TheJulia): CentOS default installs with an XFS root, and chattr
+    # fails to set +C on XFS.  This could be more elegant, however the use
+    # case is for CI testing.
+    - name: set copy-on-write for volume on non-CentOS systems
+      command: chattr +C {{ vm_volume_path }}
+      ignore_errors: yes
+      when:
+        - ansible_distribution != 'CentOS'
+        - vm_libvirt_uri == 'qemu:///system'
+
+    # Fetches the xml descriptor from the template
+    - name: create_vm
+      virt:
+        command: define
+        name: "{{ vm_name }}"
+        uri: "{{ vm_libvirt_uri }}"
+        xml: "{{ lookup('template', 'vm.xml.j2') }}"
+
+    rescue:
+      - name: "Execute `dmesg` to collect debugging output should VM creation fail."
+        command: dmesg
+      - name: >
+          "Execute `virsh capabilities` to collect debugging output
+          should VM creation fail."
+        command: virsh capabilities
+      - name: "Abort due to failed VM creation"
+        fail: >
+          msg="VM creation step failed, please review dmesg
+          output for additional details"
+    when: vm_name not in existing_vms.list_vms
+
+  # TODO(pas-ha) replace 'command: vbmc ...' tasks
+  # with a custom Ansible module using vbmc Python API
+  - name: get list of nodes from virtualbmc
+    command: vbmc list
+    register: vbmc_list
+
+  # NOTE(NobodyCam): Space at the end of the find clause is required for proper matching.
+  - name: delete vm from virtualbmc if it is there
+    command: vbmc delete {{ vm_name }}
+    when: vbmc_list.stdout.find(vm_name) != -1
+
+  - set_fact:
+      virtual_ipmi_port: "{{ (vm_ipmi_port_start|default(623) | int ) + (num_vms.stdout | int ) }}"
+
+  - name: plug vm into vbmc
+    command: vbmc add {{ vm_name }} --libvirt-uri {{ vm_libvirt_uri }} --port {{ virtual_ipmi_port }}
+
+  - name: start virtualbmc
+    command: vbmc start {{ vm_name }}
+
+  - name: get XML of the vm
+    virt:
+      name: "{{ vm_name }}"
+      command: get_xml
+    register: vm_xml
+
+  - name: Fetch the ip
+    set_fact:
+      vm_ip: "{%- for interface in item.interfaces %}{%- if 'native' in (interface.vlan | string) %}{{ interface.address }}{%- endif %}{%- endfor %}"
+
+  # Assumes there is only a single NIC per VM
+  - name: get MAC from vm XML
+    set_fact:
+      vm_mac: "{{ (vm_xml.get_xml | regex_findall(\"<mac address='.*'/>\") | first).split('=') | last | regex_replace(\"['/>]\", '') }}"
+
+  # NOTE(pas-ha) using default username and password set by virtualbmc - "admin" and "password" respectively
+  # see vbmc add --help
+  - name: set the json entry for vm
+    set_fact:
+      vm_data:
+        name: "{{ vm_name }}"
+        uuid: "{{ vm_name | to_uuid }}"
+        host_groups: "{{ vm_host_group }}"
+        driver: "{{ vm_node_driver|default('ipmi') }}"
+        driver_info:
+          power:
+            ipmi_address: "192.168.122.1"
+            ipmi_port: "{{ virtual_ipmi_port }}"
+            ipmi_username: "{{ item.remote_management.user }}"
+            ipmi_password: "{{ item.remote_management.pass }}"
+        nics:
+          - mac: "{{ vm_mac }}"
+        ansible_ssh_host: "{{ vm_ip }}"
+        ipv4_address: "{{ vm_ip }}"
+        properties:
+          cpu_arch: "{{ item.node.arch }}"
+          ram: "{{ item.node.memory.rstrip('G') }}"
+          cpus: "{{ item.node.cpus }}"
+          disk_size: "{{ item.disks[0].disk_capacity.rstrip('G') }}"
+
+  - name: add created vm info
+    set_fact:
+      nodes_json_data: "{{ nodes_json_data | combine({vm_name: vm_data}) }}"
+    when: vm_name != 'opnfv'
+
+  - name: Record OPNFV VM ip
+    set_fact:
+      opnfv_vm_ip: "{{ vm_ip }}"
+    when: vm_name == 'opnfv'
+
+  when: (num_nodes | int) > (num_vms.stdout | int)
diff --git a/xci/playbooks/roles/create-vm-nodes/tasks/download_opnfvimage.yml b/xci/playbooks/roles/create-vm-nodes/tasks/download_opnfvimage.yml
new file mode 100644
index 00000000..a227bc4f
--- /dev/null
+++ b/xci/playbooks/roles/create-vm-nodes/tasks/download_opnfvimage.yml
@@ -0,0 +1,32 @@
+---
+- name: Download the {{ xci_distro }} image checksum file
+  get_url:
+    dest: "{{ xci_cache }}/deployment_image.qcow2.sha256.txt"
+    force: no
+    url: http://artifacts.opnfv.org/releng/xci/images/{{ xci_distro }}.qcow2.sha256.txt
+    timeout: 3000
+- name: Extract checksum
+  shell: awk '{print $1}' "{{ xci_cache }}/deployment_image.qcow2.sha256.txt"
+  register: _image_checksum
+- fail:
+    msg: "Failed to get image checksum"
+  when: _image_checksum == ''
+- set_fact:
+    image_checksum: "{{ _image_checksum.stdout }}"
+- name: Download the {{ xci_distro }} image file
+  get_url:
+    url: http://artifacts.opnfv.org/releng/xci/images/{{ xci_distro }}.qcow2
+    checksum: "sha256:{{ image_checksum }}"
+    timeout: 3000
+    dest: "{{ xci_cache }}/deployment_image.qcow2"
+    force: no
+- name: Set correct mode for deployment_image.qcow2 file
+  file:
+    path: "{{ xci_cache }}/deployment_image.qcow2"
+    mode: '0755'
+    owner: 'root'
+    group: 'root'
+
+- name: Create copy of original deployment image
+  shell: "cp {{ xci_cache }}/deployment_image.qcow2 {{ opnfv_image_path }}/opnfv.qcow2"
+  become: yes
diff --git a/xci/playbooks/roles/create-vm-nodes/tasks/main.yml b/xci/playbooks/roles/create-vm-nodes/tasks/main.yml
new file mode 100644
index 00000000..7e0090e4
--- /dev/null
+++ b/xci/playbooks/roles/create-vm-nodes/tasks/main.yml
@@ -0,0 +1,49 @@
+---
+# baremetal_json_file could be the file coming from pdf/idf
+
+- name: "Load distribution defaults"
+  include_vars: "{{ ansible_os_family | lower }}.yml"
+
+# From the previous list
+- name: "Install required packages"
+  package:
+    name: "{{ required_packages }}"
+
+- include_tasks: prepare_libvirt.yml
+- include_tasks: download_opnfvimage.yml
+
+- name: create placeholder var for vm entries in JSON format
+  set_fact:
+    nodes_json_data: {}
+
+# First we create the opnfv_vm
+- include_tasks: create_vm.yml
+  with_items: "{{ [opnfv_vm] + nodes }}"
+
+- name: Start the opnfv vm
+  virt: 
+    command: start
+    name: opnfv
+
+- name: remove previous baremetal data file
+  file:
+    state: absent
+    path: "{{ baremetal_json_file }}"
+
+# We got nodes_json_data from the create_vm playbook
+- name: write to baremetal json file
+  copy:
+    dest: "{{ baremetal_json_file }}"
+    content: "{{ nodes_json_data | to_nice_json }}"
+
+- debug: var=nodes_json_data
+
+- name: >
+    "Set file permissions such that the baremetal data file
+    can be read by the user executing Ansible"
+  file:
+    path: "{{ baremetal_json_file }}"
+    owner: "{{ ansible_env.SUDO_USER }}"
+  when: >
+    ansible_env.SUDO_USER is defined and
+    baremetal_json_file != ""
diff --git a/xci/playbooks/roles/create-vm-nodes/tasks/prepare_libvirt.yml b/xci/playbooks/roles/create-vm-nodes/tasks/prepare_libvirt.yml
new file mode 100644
index 00000000..e09e2d6b
--- /dev/null
+++ b/xci/playbooks/roles/create-vm-nodes/tasks/prepare_libvirt.yml
@@ -0,0 +1,119 @@
+---
+- name: "Restart libvirt service"
+  service: name="{{libvirt_service_name}}" state=restarted
+
+# NOTE(Shrews) We need to enable ip forwarding for the libvirt bridge to
+# operate properly with dnsmasq. This should be done before starting dnsmasq.
+- name: "Enable IP forwarding in sysctl"
+  sysctl:
+    name: "net.ipv4.ip_forward"
+    value: 1
+    sysctl_set: yes
+    state: present
+    reload: yes
+
+# NOTE(Shrews) Ubuntu packaging+apparmor issue prevents libvirt from loading
+# the ROM from /usr/share/misc.
+- name: "Look for sgabios in {{ sgabios_dir }}"
+  stat: path={{ sgabios_dir }}/sgabios.bin
+  register: test_sgabios_qemu
+
+- name: "Look for sgabios in /usr/share/misc"
+  stat: path=/usr/share/misc/sgabios.bin
+  register: test_sgabios_misc
+
+- name: "Place sgabios.bin"
+  command: cp /usr/share/misc/sgabios.bin /usr/share/qemu/sgabios.bin
+  when: >
+    test_sgabios_qemu == false and
+    test_sgabios_misc == true
+
+# NOTE(TheJulia): In order to prevent conflicts, stop
+# dnsmasq to prevent conflicts with libvirt restarting.
+# TODO(TheJulia): We shouldn't need to do this, but the
+# libvirt dhcp instance conflicts withour specific config
+# and taking this path allows us to not refactor dhcp at
+# this moment. Our DHCP serving should be refactored
+# so we don't need to do this.
+- name: "Stop default dnsmasq service"
+  service:
+    name: dnsmasq
+    state: stopped
+  ignore_errors: true
+
+# NOTE(TheJulia): Seems if you test in a VM, this might
+# be helpful if your installed your host originally
+# with the default 192.168.122/0/24 network
+- name: destroy libvirt network
+  virt_net:
+    name: "{{ vm_network }}"
+    state: absent
+    uri: "{{ vm_libvirt_uri }}"
+
+- name: ensure libvirt network is present
+  virt_net:
+    name: "{{ vm_network }}"
+    state: present
+    xml: "{{ lookup('template', 'net.xml.j2') }}"
+    uri: "{{ vm_libvirt_uri }}"
+
+- name: find facts on libvirt networks
+  virt_net:
+    command: facts
+    uri: "{{ vm_libvirt_uri }}"
+
+# NOTE(pas-ha) yet another place where non-local libvirt will not work
+- name: "Delete network interface if virtual network is not active"
+  command: ip link del {{ ansible_libvirt_networks[vm_network].bridge }}
+  when:
+    - ansible_libvirt_networks[vm_network].state != 'active'
+    - vm_libvirt_uri == 'qemu:///system'
+  ignore_errors: yes
+
+- name: set libvirt network to autostart
+  virt_net:
+    name: "{{ vm_network }}"
+    autostart: yes
+    uri: "{{ vm_libvirt_uri }}"
+
+- name: ensure libvirt network is running
+  virt_net:
+    name: "{{ vm_network }}"
+    state: active
+    uri: "{{ vm_libvirt_uri }}"
+
+- name: get libvirt network status
+  virt_net:
+    name: "{{ vm_network }}"
+    command: status
+    uri: "{{ vm_libvirt_uri }}"
+  register: test_vm_net_status
+
+- name: fail if libvirt network is not active
+  assert:
+    that: test_vm_net_status.status == 'active'
+
+- name: define a libvirt pool if not set
+  virt_pool:
+    name: "{{ node_storage_pool }}"
+    state: present
+    uri: "{{ vm_libvirt_uri }}"
+    xml: "{{ lookup('template', 'pool_dir.xml.j2') }}"
+
+- name: ensure libvirt pool is running
+  virt_pool:
+    name: "{{ node_storage_pool }}"
+    state: active
+    autostart: yes
+    uri: "{{ vm_libvirt_uri }}"
+
+- name: create dir for bm logs
+  file:
+    state: directory
+    path: "{{ node_logdir }}"
+    recurse: yes
+    mode: "0755"
+
+- name: install virtualbmc
+  pip:
+    name: virtualbmc