Migrate install.sh script to ansible

Ansible scripts to install yardstick.

These scripts will take a few options:
  INSTALLATION_MODE : Container or Baremetal
  YARDSTICK_DIR: Location of Yardstick directory
  VIRTUAL_ENVIRONMENT: Should a virtual environment be used
  NSB_DIR: Location of NSB directory

JIRA: YARDSTICK-1126

Change-Id: I4a13e07e96650ffd971dfb18472f2563cfd51aa9
Signed-off-by: John O Loughlin <john.oloughlin@intel.com>

17 files changed, 496 insertions, 23 deletions

diff --git a/ansible/roles/add_repos_jumphost/tasks/Debian.yml b/ansible/roles/add_repos_jumphost/tasks/Debian.yml
new file mode 100644
index 000000000..0b67c6691
--- /dev/null
+++ b/ansible/roles/add_repos_jumphost/tasks/Debian.yml
@@ -0,0 +1,82 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# Arguments needed: arch_arm64, arch_amd64, ubuntu_archive
+
+- name: Set the repositories architecture name
+  set_fact:
+    arch: "{{ arch_arm64 if ansible_architecture == 'aarch64' else arch_amd64 }}"
+    extra_arch: "{{ arch_amd64 if ansible_architecture == 'aarch64' else arch_arm64 }}"
+
+- name: Define the repositories names
+  set_fact:
+    repo: "{{ ubuntu_archive[arch] }}"
+    extra_repo: "{{ ubuntu_archive[extra_arch] }}"
+
+- name: Add architecture to the default repository list
+  replace:
+    path: "{{ sources_list_file }}"
+    regexp: '(^deb\s+)([^\[].*)$'
+    replace: 'deb [arch={{ arch }}] \2'
+
+- name: Remove support for source repositories
+  replace:
+    path: "{{ sources_list_file }}"
+    regexp: "^deb-src "
+    replace: "# deb-src "
+
+- name: Add extra architecture
+  apt:
+    dpkg_options: "add-architecture {{ extra_arch }}"
+
+- name: Define the default release version
+  copy:
+    dest: "{{ default_distro_file }}"
+    content: 'APT::Default-Release "{{ ansible_distribution_release }}";'
+
+- name: Remove extra repository file
+  file:
+    path: "{{ repo_file }}"
+    state: absent
+  ignore_errors: yes
+
+- name: Add extra repository file
+  file:
+    path: "{{ repo_file }}"
+    state: touch
+
+- name: Add the repository for qemu_static_user/xenial
+  blockinfile:
+    path: "{{ repo_file }}"
+    marker: "MARKER"
+    content: |
+      deb [arch={{ arch }}] {{ repo }} xenial-updates universe
+  when: ansible_distribution_release != "xenial"
+
+- name: Add extra architecture repositories if installing in container
+  blockinfile:
+    path: "{{ repo_file }}"
+    marker: "MARKER"
+    content: |
+      deb [arch={{ extra_arch }}] {{ extra_repo }} {{ ansible_distribution_release }} main universe multiverse restricted
+      deb [arch={{ extra_arch }}] {{ extra_repo }} {{ ansible_distribution_release }}-updates main universe multiverse restricted
+      deb [arch={{ extra_arch }}] {{ extra_repo }} {{ ansible_distribution_release }}-security main universe multiverse restricted
+      deb [arch={{ extra_arch }}] {{ extra_repo }} {{ ansible_distribution_release }}-proposed main universe multiverse restricted
+  when: installation_mode == "container"
+
+- name: Remove the marker
+  lineinfile:
+    dest: "{{ repo_file }}"
+    state: absent
+    regexp: "MARKER"
diff --git a/ansible/roles/add_repos_jumphost/tasks/main.yml b/ansible/roles/add_repos_jumphost/tasks/main.yml
new file mode 100644
index 000000000..f50fd9f0d
--- /dev/null
+++ b/ansible/roles/add_repos_jumphost/tasks/main.yml
@@ -0,0 +1,16 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- include: "{{ ansible_os_family }}.yml"
+  when: ansible_os_family == "Debian"
diff --git a/ansible/roles/add_repos_jumphost/vars/main.yml b/ansible/roles/add_repos_jumphost/vars/main.yml
new file mode 100644
index 000000000..30e444711
--- /dev/null
+++ b/ansible/roles/add_repos_jumphost/vars/main.yml
@@ -0,0 +1,17 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+repo_file: "/etc/apt/sources.list.d/yardstick.list"
+sources_list_file: "/etc/apt/sources.list"
+default_distro_file: "/etc/apt/apt.conf.d/default-distro"
diff --git a/ansible/roles/configure_gui/tasks/main.yml b/ansible/roles/configure_gui/tasks/main.yml
new file mode 100644
index 000000000..846a9cb47
--- /dev/null
+++ b/ansible/roles/configure_gui/tasks/main.yml
@@ -0,0 +1,33 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Define variables
+  set_fact:
+    gui_dir: "{{ yardstick_dir }}/gui/"
+
+- name: Run gui.sh
+  shell:
+    cmd: /bin/bash gui.sh
+    chdir: "{{ gui_dir }}"
+
+- name: Create nginx/yardstick directory
+  file:
+    path: /etc/nginx/yardstick
+    state: directory
+    recurse: yes
+
+- name: Move dist to /etc/nginx/yardstick/gui
+  shell:
+    cmd: mv dist /etc/nginx/yardstick/gui
+    chdir: "{{ gui_dir }}"
diff --git a/ansible/roles/configure_nginx/tasks/main.yml b/ansible/roles/configure_nginx/tasks/main.yml
new file mode 100644
index 000000000..37b052725
--- /dev/null
+++ b/ansible/roles/configure_nginx/tasks/main.yml
@@ -0,0 +1,33 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Define variables
+  set_fact:
+    socket_file: "{{ socket_file|default('/var/run/yardstick.sock') }}"
+
+- name: Make sure conf.d directory exists
+  file:
+    path: /etc/nginx/conf.d
+    state: directory
+
+- name: Create the nginx config file
+  template:
+    src: yardstick.conf.j2
+    dest: "/etc/nginx/conf.d/yardstick.conf"
+
+- name: Configure ports if RedHat OS
+  shell: |
+    semanage port -m -t http_port_t  -p tcp 5000
+    semanage port -m -t http_port_t  -p udp 5000
+  when: ansible_os_family == "RedHat"
\ No newline at end of file
diff --git a/ansible/roles/configure_nginx/templates/yardstick.conf.j2 b/ansible/roles/configure_nginx/templates/yardstick.conf.j2
new file mode 100644
index 000000000..484096cec
--- /dev/null
+++ b/ansible/roles/configure_nginx/templates/yardstick.conf.j2
@@ -0,0 +1,18 @@
+server {
+    listen 5000;
+    server_name localhost;
+    index index.htm index.html;
+    location / {
+        include uwsgi_params;
+        client_max_body_size    2000m;
+        uwsgi_pass unix://{{ socket_file }};
+    }
+
+    location /gui/ {
+        alias /etc/nginx/yardstick/gui/;
+    }
+
+    location /report/ {
+        alias /tmp/;
+    }
+}
diff --git a/ansible/roles/configure_rabbitmq/tasks/main.yml b/ansible/roles/configure_rabbitmq/tasks/main.yml
new file mode 100644
index 000000000..3ad60c1ea
--- /dev/null
+++ b/ansible/roles/configure_rabbitmq/tasks/main.yml
@@ -0,0 +1,30 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Restart rabbitmq
+  service:
+    name: rabbitmq-server
+    state: restarted
+
+- name: rabbitmqctl start_app
+  shell: rabbitmqctl start_app
+
+- name: Configure rabbitmq
+  rabbitmq_user:
+    user: yardstick
+    password: yardstick
+    configure_priv: .*
+    read_priv: .*
+    write_priv: .*
+    state: present
diff --git a/ansible/roles/configure_uwsgi/tasks/main.yml b/ansible/roles/configure_uwsgi/tasks/main.yml
new file mode 100644
index 000000000..6a2244657
--- /dev/null
+++ b/ansible/roles/configure_uwsgi/tasks/main.yml
@@ -0,0 +1,45 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Define variables
+  set_fact:
+    config_dir: "/etc/yardstick/"
+    log_dir: "/var/log/yardstick/"
+    socket_file: "/var/run/yardstick.sock"
+
+- name: Create UWSGI config directory
+  file:
+    path: "/etc/yardstick"
+    state: directory
+    owner: root
+    mode: 0755
+
+- name: Create API log directory
+  file:
+    path: "{{ log_dir }}"
+    state: directory
+    owner: root
+    mode: 0777
+
+- name: Create the socket for communicating
+  file:
+    path: "{{ socket_file }}"
+    state: touch
+    owner: root
+    mode: 0644
+
+- name: Create the UWSGI config file
+  template:
+    src: yardstick.ini.j2
+    dest: "{{ config_dir }}yardstick.ini"
diff --git a/ansible/roles/configure_uwsgi/templates/yardstick.ini.j2 b/ansible/roles/configure_uwsgi/templates/yardstick.ini.j2
new file mode 100644
index 000000000..c049daf84
--- /dev/null
+++ b/ansible/roles/configure_uwsgi/templates/yardstick.ini.j2
@@ -0,0 +1,18 @@
+[uwsgi]
+master = true
+debug = true
+chdir = {{ yardstick_dir }}api
+module = server
+plugins = python
+processes = 10
+threads = 5
+async = true
+max-requests = 5000
+chmod-socket = 666
+callable = app_wrapper
+enable-threads = true
+close-on-exec = 1
+daemonize = {{ log_dir }}uwsgi.log
+socket = {{ socket_file }}
+{# If virtual environment, we need to add:
+   virtualenv = <virtual_env> #}
\ No newline at end of file
diff --git a/ansible/roles/download_trex/tasks/main.yml b/ansible/roles/download_trex/tasks/main.yml
index baa964fd8..9df67d939 100644
--- a/ansible/roles/download_trex/tasks/main.yml
+++ b/ansible/roles/download_trex/tasks/main.yml
@@ -12,6 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- name: make sure trex_dest exists
+  file:
+    path: "{{ trex_dest }}"
+    state: directory
+
 - name: fetch Trex
   get_url:
     url: "{{ trex_url }}"
diff --git a/ansible/roles/install_dependencies_jumphost/tasks/Debian.yml b/ansible/roles/install_dependencies_jumphost/tasks/Debian.yml
new file mode 100755
index 000000000..9baf7e59e
--- /dev/null
+++ b/ansible/roles/install_dependencies_jumphost/tasks/Debian.yml
@@ -0,0 +1,76 @@
+# Copyright (c) 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Update repositories
+  apt:
+    update_cache: yes
+
+- name: Install core packages
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items:
+    - wget
+    - curl
+    - screen
+    - procps
+    - socat
+    - sshpass
+    - sudo
+    - vim
+    - libffi-dev
+    - libfuse-dev
+    - libssl-dev
+    - libxft-dev
+    - libxml2-dev
+    - libxss-dev
+    - libxslt-dev
+    - libxslt1-dev
+    - libzmq-dev
+    - qemu-user-static
+    - qemu-utils
+    - kpartx
+    - python
+    - python-setuptools
+    - python-dev
+    - python-pip
+    - python-libvirt
+    - python-virtualenv
+    - bridge-utils
+    - ebtables
+    - openssl
+    - ccze
+    - nginx-full
+    - uwsgi
+    - uwsgi-plugin-python
+    - supervisor
+    - lsof
+    - nodejs
+    - npm
+    - rabbitmq-server
+
+- name: Install libc6:arm64 package
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items:
+    - libc6:arm64
+  when:
+    - arch is defined
+    - arch != arch_arm64
+    - installation_mode == inst_mode_container
+
+- name: Remove dependencies that are no longer required
+  apt:
+    update_cache: yes
+
+- name: Remove useless packages from the cache
+  apt:
+    autoclean: yes
diff --git a/ansible/roles/install_dependencies/tasks/RedHat.yml b/ansible/roles/install_dependencies_jumphost/tasks/RedHat.yml
index a5d4d0b15..85eb1156a 100644
--- a/ansible/roles/install_dependencies/tasks/RedHat.yml
+++ b/ansible/roles/install_dependencies_jumphost/tasks/RedHat.yml
@@ -42,5 +42,13 @@
     - python-setuptools
     - libffi-devel
     - python-devel
-    - kpartx
-
+    - nodejs
+    - npm
+    - gcc
+    - lsof
+    - procps
+    - bridge-utils
+    - ebtables
+    - openssl
+    - python-virtualenv
+    - ccze
diff --git a/ansible/roles/install_dependencies/tasks/Debian.yml b/ansible/roles/install_dependencies_jumphost/tasks/Suse.yml
index bba6fb13c..af53c9cd5 100755..100644
--- a/ansible/roles/install_dependencies/tasks/Debian.yml
+++ b/ansible/roles/install_dependencies_jumphost/tasks/Suse.yml
@@ -1,4 +1,4 @@
-# Copyright (c) 2017 Intel Corporation.
+# Copyright (c) 2018 Intel Corporation.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,39 +12,38 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ---
+- name: Install EPEL if needed
+  action: "{{ ansible_pkg_mgr }} name=epel-release state=present"
+  when: ansible_distribution in ['RedHat', 'CentOS', 'ScientificLinux']
+
 - name: Install core packages
   action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
   with_items:
-    - python-minimal
+    - deltarpm
     - wget
+    - expect
     - curl
     - screen
-    - procps
     - git
     - socat
     - sshpass
-    - libxslt1-dev
-    - libffi-dev
-    - libfuse-dev
     - qemu-kvm
-    - qemu-user-static
-    - qemu-utils
     - kpartx
-    - libvirt0
-    - python-libvirt
+    - libxslt-devel
+    - libffi-devel
+    - openssl-devel
+    - nginx
+    - uwsgi
+    - python-setuptools
+    - libffi-devel
+    - python-devel
+    - nodejs
+    - npm
+    - gcc
+    - lsof
+    - procps
     - bridge-utils
     - ebtables
     - openssl
-    - libssl-dev
-    - python-dev
     - python-virtualenv
     - ccze
-    - libxml2-dev
-    - libxslt-dev
-    - libzmq-dev
-    - nginx-full
-    - uwsgi
-    - uwsgi-plugin-python
-    - supervisor
-    - python-setuptools
-    - lsof
diff --git a/ansible/roles/install_dependencies/tasks/main.yml b/ansible/roles/install_dependencies_jumphost/tasks/main.yml
index 27660c3ca..27660c3ca 100644
--- a/ansible/roles/install_dependencies/tasks/main.yml
+++ b/ansible/roles/install_dependencies_jumphost/tasks/main.yml
diff --git a/ansible/roles/install_yardstick/tasks/main.yml b/ansible/roles/install_yardstick/tasks/main.yml
new file mode 100644
index 000000000..ee1b83756
--- /dev/null
+++ b/ansible/roles/install_yardstick/tasks/main.yml
@@ -0,0 +1,46 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+# Arguments needed: map_min_addr_file, yardstick_dir
+
+- name: Define variables
+  set_fact:
+    map_min_addr_file: "/etc/sysctl.d/mmap_min_addr.conf"
+
+- name: Remove the kernel minimum virtual address restriction that a process is allowed to mmap
+  copy:
+    dest: "{{ map_min_addr_file }}"
+    content: "vm.mmap_min_addr = 0\n"
+
+- name: Config git SSL
+  git_config:
+    name: http.sslVerify
+    scope: global
+    value: False
+
+# There is a bug with the easy install ansible module in suse linux.
+# Until this is fixed the shell command must be used
+- name: Install pip
+  shell: easy_install -U pip
+#    easy_install:
+#    name: pip
+#    state: latest
+
+- name: install yardstick without virtual environment
+  include_tasks: regular_install.yml
+  when: virtual_environment == False
+
+- name: install yardstick with virtual environment
+  include_tasks: virtual_install.yml
+  when: virtual_environment == True
diff --git a/ansible/roles/install_yardstick/tasks/regular_install.yml b/ansible/roles/install_yardstick/tasks/regular_install.yml
new file mode 100644
index 000000000..4a9925ab4
--- /dev/null
+++ b/ansible/roles/install_yardstick/tasks/regular_install.yml
@@ -0,0 +1,22 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Install Yardstick requirements
+  pip:
+    requirements: "{{ yardstick_dir }}/requirements.txt"
+
+- name: Install Yardstick code
+  pip:
+    name: "{{ yardstick_dir }}/."
+    extra_args: -e
diff --git a/ansible/roles/install_yardstick/tasks/virtual_install.yml b/ansible/roles/install_yardstick/tasks/virtual_install.yml
new file mode 100644
index 000000000..8545acbcb
--- /dev/null
+++ b/ansible/roles/install_yardstick/tasks/virtual_install.yml
@@ -0,0 +1,25 @@
+# Copyright (c) 2018 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Install Yardstick requirements
+  pip:
+    requirements: "{{ yardstick_dir }}/requirements.txt"
+    virtualenv: "{{ yardstick_dir }}/virtualenv"
+
+- name: Install Yardstick code
+  pip:
+    name: "{{ yardstick_dir }}/."
+    extra_args: -e
+    virtualenv: "{{ yardstick_dir }}/virtualenv"
+