Add dpdk plugin

JIRA: COMPASS-550

add OVS-DPDK plugin, there are two roles in this plugin.
    1. ins_dpdk is for dpdk installation and hugepages setting.
    2. ins_ovs is for openvswitch installation.
add dpdk scenario networking configuration example called "network_cfg_dpdk.yaml"

Change-Id: Ifd8c1aadc218753f99bc26bb530e7cf9962ad8e3
Signed-off-by: chigang <chigang@huawei.com>

16 files changed, 509 insertions, 0 deletions

diff --git a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
index e1efebfa..49f76cc7 100644
--- a/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
+++ b/deploy/adapters/ansible/openstack/HA-ansible-multinodes.yml
@@ -23,6 +23,8 @@
     - config-compute
     - storage
     - rt_kvm
+    - ins_dpdk
+    - ins_ovs
 
 - hosts: all
   remote_user: root
diff --git a/deploy/conf/network_cfg_dpdk.yaml b/deploy/conf/network_cfg_dpdk.yaml
new file mode 100644
index 00000000..e196d6f4
--- /dev/null
+++ b/deploy/conf/network_cfg_dpdk.yaml
@@ -0,0 +1,132 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+nic_mappings: []
+bond_mappings: []
+
+provider_net_mappings:
+  - name: br-provider
+    network: physnet
+    interface: eth10
+    type: ovs
+    role:
+      - controller
+
+sys_intf_mappings:
+  - name: mgmt
+    interface: eth0
+    type: normal
+    vlan_tag: None
+    role:
+      - controller
+      - compute
+
+  - name: tenant
+    interface: eth2
+    type: normal
+    vlan_tag: None
+    role:
+      - controller
+
+  - name: tenant
+    interface: eth2
+    type: dpdk
+    vlan_tag: None
+    role:
+      - compute
+
+  - name: storage
+    interface: eth1
+    type: normal
+    vlan_tag: 102
+    role:
+      - controller
+      - compute
+
+  - name: external
+    interface: eth1
+    type: normal
+    vlan_tag: None
+    role:
+      - controller
+      - compute
+
+ip_settings:
+  - name: mgmt
+    ip_ranges:
+      - - "10.1.0.50"
+        - "10.1.0.100"
+    dhcp_ranges:
+      - - "10.1.0.2"
+        - "10.1.0.49"
+    cidr: "10.1.0.0/24"
+    gw: "10.1.0.1"
+    role:
+      - controller
+      - compute
+
+  - name: tenant
+    ip_ranges:
+      - - "172.16.1.1"
+        - "172.16.1.50"
+    cidr: "172.16.1.0/24"
+    role:
+      - controller
+      - compute
+
+  - name: storage
+    ip_ranges:
+      - - "172.16.2.1"
+        - "172.16.2.50"
+    cidr: "172.16.2.0/24"
+    role:
+      - controller
+      - compute
+
+  - name: external
+    ip_ranges:
+      - - "192.16.1.210"
+        - "192.16.1.220"
+    cidr: "192.16.1.0/24"
+    gw: "192.16.1.1"
+    role:
+      - controller
+      - compute
+
+internal_vip:
+  ip: 10.1.0.222
+  netmask: "24"
+  interface: mgmt
+
+public_vip:
+  ip: 192.16.1.222
+  netmask: "24"
+  interface: external
+
+onos_nic: eth2
+tenant_net_info:
+  type: vxlan
+  range: "1:1000"
+  provider_network: None
+
+public_net_info:
+  enable: "True"
+  network: ext-net
+  type: flat
+  segment_id: 1000
+  subnet: ext-subnet
+  provider_network: physnet
+  router: router-ext
+  enable_dhcp: "False"
+  no_gateway: "False"
+  external_gw: "192.16.1.1"
+  floating_ip_cidr: "192.16.1.0/24"
+  floating_ip_start: "192.16.1.101"
+  floating_ip_end: "192.16.1.199"
diff --git a/deploy/conf/vm_environment/os-nosdn-dpdk-ha.yml b/deploy/conf/vm_environment/os-nosdn-dpdk-ha.yml
index d098afba..3cd05faf 100644
--- a/deploy/conf/vm_environment/os-nosdn-dpdk-ha.yml
+++ b/deploy/conf/vm_environment/os-nosdn-dpdk-ha.yml
@@ -11,6 +11,9 @@
 TYPE: virtual
 FLAVOR: cluster
 
+plugins:
+  - dpdk: "Enable"
+
 hosts:
   - name: host1
     roles:
diff --git a/deploy/conf/vm_environment/os-nosdn-dpdk-noha.yml b/deploy/conf/vm_environment/os-nosdn-dpdk-noha.yml
index be1d9510..9fc2ca3f 100644
--- a/deploy/conf/vm_environment/os-nosdn-dpdk-noha.yml
+++ b/deploy/conf/vm_environment/os-nosdn-dpdk-noha.yml
@@ -11,6 +11,9 @@
 TYPE: virtual
 FLAVOR: cluster
 
+plugins:
+  - dpdk: "Enable"
+
 hosts:
   - name: host1
     roles:
diff --git a/plugins/dpdk/plugins.desc b/plugins/dpdk/plugins.desc
new file mode 100644
index 00000000..9989af5e
--- /dev/null
+++ b/plugins/dpdk/plugins.desc
@@ -0,0 +1,67 @@
+# This dpdk plugin adds ovs-dpdk and dpdk features from ovsnfv project
+# into Compass4nfv together with scenarios.
+#
+#
+# More details can be found in the development document.
+# ##############################################################
+---
+plugin:
+  # plugin name,it is also as the switch to enable/disable plugin in scenario
+  # files
+  name: dpdk
+
+  description: ovs-dpdk and dpdk based on ovsnfv
+
+  maintainers:
+    - david.j.chou@intel.com
+    - chigang@huawei.com
+
+  # host os type: ubuntu/centos
+  os_version: ubuntu
+
+  # true: this plugin is deployed separately on a new node
+  # false: this plugin is deployed on controller or compute node
+  independent_hosts: false
+
+  # artifact: packege download url for this plugin
+  artifacts:
+    url:
+
+  global_vars:
+    - dpdk_repo: https://github.com/dpdk/dpdk.git
+    - dpdk_version: v16.11
+    - ovs_repo: https://github.com/openvswitch/ovs.git
+    - ovs_version: v2.7.2
+
+  # orchestration
+  # A plugin can have mutiple components, each component may need to be
+  # installed on different inventory or have its own configuration.
+  # due to Compass4nfv currently only supports ansible, so each component
+  # of the installation and configuration script need to be use ansible.
+  # cm : congfiguration management tool : only ansible support
+  # role: each component corresponds to ansible script that locates in the same
+  # directory as plugin.desc.
+  # phrase: pre_openstack -- the component is installed after the OS
+  # provisioning, before the OpenStack deployment.
+  # phrase: post_openstack -- the component is installed before the OpenStack
+  # deployment.
+  # inventory: if the phrase is pre_openstack, inventory can be controller and
+  # compute. if the phrase is post_openstack, inventory can be get from the file
+  # openstack-ansible.inventory
+  orchestration:
+    cm: ansible
+    roles:
+      - role: ins_dpdk
+        phrase: pre_openstack
+        inventory:
+          - compute
+
+      - role: ins_ovs
+        phrase: pre_openstack
+        inventory:
+          - compute
+
+     - role: dpdk_config
+        phrase: post_openstack
+        inventory:
+          - neutron_agents_container
diff --git a/plugins/dpdk/roles/ins_dpdk/tasks/dpdk.yml b/plugins/dpdk/roles/ins_dpdk/tasks/dpdk.yml
new file mode 100644
index 00000000..6a08386f
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/tasks/dpdk.yml
@@ -0,0 +1,81 @@
+# #############################################################################
+# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# #############################################################################
+---
+- name: install prerequisites package
+  apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - git
+    - gcc
+    - make
+    - cmake
+    - libpcap0.8
+    - libpcap0.8-dev
+
+- name: git clone DPDK code
+  git:
+    repo: "{{ dpdk_repo }}"
+    dest: "{{ dpdk_dir }}"
+    version: "{{ dpdk_version }}"
+
+- name: Configure DPDK
+  lineinfile:
+    dest: "{{ dpdk_dir }}/config/common_linuxapp"
+    regexp: '^{{ item.key }}='
+    line: "{{ item.key }}={{ item.value }}"
+  with_items:
+    - key: CONFIG_RTE_BUILD_COMBINE_LIBS
+      value: y
+  register: dpdk_config_change
+
+- name: make config DPDK
+  command: make config T=x86_64-native-linuxapp-gcc chdir={{ dpdk_dir }}
+
+- name: Configure PMD
+  lineinfile:
+    dest: "{{ dpdk_dir }}/build/.config"
+    regexp: '^{{ item.key }}='
+    line: "{{ item.key }}={{ item.value }}"
+  with_items:
+    - key: PMD_PCAP
+      value: y
+
+- name: Check if DPDK build exists
+  stat: path={{ dpdk_build }}
+  register: dpdk_build_status
+
+- name: Build DPDK
+  command: make install T=x86_64-native-linuxapp-gcc chdir={{ dpdk_dir }}
+  when: (dpdk_build_status.stat.isdir is not defined) or
+        (dpdk_rebuild is defined) or
+        dpdk_config_change.changed or dpdk_changed.changed
+
+- name: Get dpdk interface device name
+  command: echo "{{ compu_sys_mappings['tenant']['interface'] }}"
+  register: dpdk_device_name
+  when: compu_sys_mappings["tenant"]["type"] == "dpdk"
+
+- debug:
+    msg: "{{ dpdk_device_name }}"
+
+- name: Create DPDK scripts
+  template:
+    src: "templates/{{ item.name }}.j2"
+    dest: "/root/{{ item.name }}"
+    mode: 0755
+  with_items:
+    - name: dpdk_uio.sh
+      dpdk_build: '{{ dpdk_build }}'
+      dpdk_dir: '{{ dpdk_dir }}'
+      device_name: '{{ dpdk_device_name.stdout | default("eth2") }}'
+    - name: dpdk_vfio.sh
+      dpdk_build: '{{ dpdk_build }}'
+      dpdk_dir: '{{ dpdk_dir }}'
+      device_name: '{{ dpdk_device_name.stdout | default("eth2") }}'
diff --git a/plugins/dpdk/roles/ins_dpdk/tasks/hugepages.yml b/plugins/dpdk/roles/ins_dpdk/tasks/hugepages.yml
new file mode 100644
index 00000000..3f41cf03
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/tasks/hugepages.yml
@@ -0,0 +1,45 @@
+# #############################################################################
+# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# #############################################################################
+---
+# yamllint disable rule:truthy
+- name: Check hugepages
+  shell: grep -q intel_iommu /etc/default/grub
+  register: check_result
+  ignore_errors: True
+# yamllint enable rule:truthy
+
+- name: Config grub
+  lineinfile:
+    dest: /etc/default/grub
+    regexp: '^GRUB_CMDLINE_LINUX_DEFAULT=""'
+    line: "{{ grub_cmdline }}"
+    state: present
+  when: check_result.rc == 1
+
+- name: Update grub
+  shell: update-grub
+  when: check_result.rc == 1
+
+- name: wait a moment
+  command: sleep 5
+  when: check_result.rc == 1
+
+- name: Reboot
+  shell: sleep 2 && shutdown -r now 'Reboot required'
+  become: true
+  async: 1
+  poll: 0
+  when: check_result.rc == 1
+  ignore_errors: true
+
+- name: Wait for reboot
+  local_action:
+    module: wait_for
+      host={{ ansible_eth0.ipv4.address }} port=22 delay=1 timeout=300
+  when: check_result.rc == 1
diff --git a/plugins/dpdk/roles/ins_dpdk/tasks/main.yml b/plugins/dpdk/roles/ins_dpdk/tasks/main.yml
new file mode 100644
index 00000000..923dc3b4
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/tasks/main.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- include: hugepages.yml
+  when: dpdk is defined and dpdk == "Enable"
+
+- include: dpdk.yml
+  when: dpdk is defined and dpdk == "Enable"
diff --git a/plugins/dpdk/roles/ins_dpdk/templates/dpdk_uio.sh.j2 b/plugins/dpdk/roles/ins_dpdk/templates/dpdk_uio.sh.j2
new file mode 100644
index 00000000..560d30ca
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/templates/dpdk_uio.sh.j2
@@ -0,0 +1,9 @@
+#!/bin/bash
+ifdown {{ item.device_name }}
+modprobe uio
+lsmod |grep igb_uio
+if [ $? == 0 ];then
+    rmmod igb_uio
+fi
+insmod {{ item.dpdk_build }}/kmod/igb_uio.ko
+{{ item.dpdk_dir }}/tools/dpdk-devbind.py --bind=igb_uio {{ item.device_name }}
diff --git a/plugins/dpdk/roles/ins_dpdk/templates/dpdk_vfio.sh.j2 b/plugins/dpdk/roles/ins_dpdk/templates/dpdk_vfio.sh.j2
new file mode 100644
index 00000000..58839342
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/templates/dpdk_vfio.sh.j2
@@ -0,0 +1,6 @@
+#!/bin/bash
+ifdown {{ item.device_name }}
+modprobe vfio-pci
+chmod a+x /dev/vfio
+chmod 0666 /dev/vfio/*
+{{ item.dpdk_dir }}/tools/dpdk-devbind.py --bind=vfio-pci {{ item.device_name }}
diff --git a/plugins/dpdk/roles/ins_dpdk/vars/main.yml b/plugins/dpdk/roles/ins_dpdk/vars/main.yml
new file mode 100644
index 00000000..87e1026c
--- /dev/null
+++ b/plugins/dpdk/roles/ins_dpdk/vars/main.yml
@@ -0,0 +1,17 @@
+##############################################################################
+# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+# yamllint disable rule:line-length
+grub_cmdline: GRUB_CMDLINE_LINUX_DEFAULT="hugepagesz=2M hugepages=2048 iommu=pt intel_iommu=on isolcpus=1,2"
+# yamllint enable rule:line-length
+
+dpdk_dir: /root/dpdk
+dpdk_build: '{{ dpdk_dir }}/x86_64-native-linuxapp-gcc'
+dpdk_repo: https://github.com/dpdk/dpdk.git
+dpdk_version: v16.11
diff --git a/plugins/dpdk/roles/ins_ovs/tasks/main.yml b/plugins/dpdk/roles/ins_ovs/tasks/main.yml
new file mode 100644
index 00000000..6570d06c
--- /dev/null
+++ b/plugins/dpdk/roles/ins_ovs/tasks/main.yml
@@ -0,0 +1,11 @@
+# #############################################################################
+# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# #############################################################################
+---
+- include: ovs.yml
+  when: dpdk is defined and dpdk == "Enable"
diff --git a/plugins/dpdk/roles/ins_ovs/tasks/ovs.yml b/plugins/dpdk/roles/ins_ovs/tasks/ovs.yml
new file mode 100644
index 00000000..9f728696
--- /dev/null
+++ b/plugins/dpdk/roles/ins_ovs/tasks/ovs.yml
@@ -0,0 +1,97 @@
+# #############################################################################
+# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+# #############################################################################
+---
+- name: install prerequisites package
+  apt:
+    name: "{{ item }}"
+    state: present
+  with_items:
+    - git
+    - libnuma-dev
+    - dh-autoreconf
+    - python-pip
+
+- name: git clone open-vswitch code
+  git:
+    repo: "{{ ovs_repo }}"
+    dest: "{{ ovs_dir }}"
+    version: "{{ ovs_version }}"
+
+- name: install prerequisites package
+  pip:
+    name: six
+
+- name: Check if OVS configure script exists
+  stat: path={{ ovs_dir }}/configure
+  register: ovs_config_status
+
+- name: Bootstrap OVS if required
+  command: ./boot.sh chdir={{ ovs_dir }}
+  when: ovs_config_status.stat.exists == false or (ovs_rebuild is defined) or ovs_changed.changed
+
+- name: Check if OVS Makefile exists
+  stat: path={{ ovs_dir }}/Makefile
+  register: ovs_makefile_status
+
+# yamllint disable rule:line-length
+- name: Configure OVS
+  command: ./configure --with-dpdk={{ dpdk_build }} CFLAGS="-g -O2 -Wno-cast-align" chdir={{ ovs_dir }}
+  when: ovs_makefile_status.stat.exists == false or (ovs_rebuild is defined) or ovs_changed.changed
+# yamllint enable rule:line-length
+
+- name: Check if OVS distribution files exists
+  stat: path={{ ovs_dir }}/distfiles
+  register: ovs_distfiles_status
+
+- name: Build OVS
+  command: make CFLAGS='-O3 -march=native' chdir={{ ovs_dir }}
+  when: ovs_distfiles_status.stat.exists == false or (ovs_rebuild is defined) or ovs_changed.changed
+
+- name: Check if OVS tools are installed
+  stat: path=/usr/local/bin/ovsdb-tool
+  register: ovs_tools_status
+
+- name: Install OVS tools
+  command: make install chdir={{ ovs_dir }}
+  when: ovs_tools_status.stat.exists == false or (ovs_rebuild is defined) or ovs_changed.changed
+
+- name: Create OVS scripts
+  template:
+    src: "templates/{{ item }}.j2"
+    dest: "/root/{{ item }}"
+    mode: 0755
+  with_items:
+    - start_ovs_vswitchd.sh
+    - start_ovsdb_server.sh
+
+- name: Create folders
+  file: path={{ item }} state=directory
+  with_items:
+    - /usr/local/etc/openvswitch
+    - /usr/local/var/run/openvswitch
+
+- name: Clear database configuration if required
+  file: path=/usr/local/etc/openvswitch/conf.db state=absent
+  when: ovs_rebuild is defined or ovs_changed.changed
+
+- name: Check if database configuration exists
+  stat: path=/usr/local/etc/openvswitch/conf.db
+  register: ovs_dbconfig_status
+
+# yamllint disable rule:line-length
+- name: Create database configuration
+  command: ovsdb-tool create /usr/local/etc/openvswitch/conf.db /usr/local/share/openvswitch/vswitch.ovsschema
+  when: ovs_dbconfig_status.stat.exists == false
+# yamllint enable rule:line-length
+
+- name: Start ovsdb Server
+  command: /root/start_ovsdb_server.sh
+
+- name: Start OVS vswitchd with DPDK support enabled
+  command: /root/start_ovs_vswitchd.sh
diff --git a/plugins/dpdk/roles/ins_ovs/templates/start_ovs_vswitchd.sh.j2 b/plugins/dpdk/roles/ins_ovs/templates/start_ovs_vswitchd.sh.j2
new file mode 100644
index 00000000..c1504177
--- /dev/null
+++ b/plugins/dpdk/roles/ins_ovs/templates/start_ovs_vswitchd.sh.j2
@@ -0,0 +1,4 @@
+#!/bin/bash
+export DB_SOCK=/usr/local/var/run/openvswitch/db.sock
+sudo ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
+sudo ovs-vswitchd unix:$DB_SOCK --pidfile --detach
diff --git a/plugins/dpdk/roles/ins_ovs/templates/start_ovsdb_server.sh.j2 b/plugins/dpdk/roles/ins_ovs/templates/start_ovsdb_server.sh.j2
new file mode 100644
index 00000000..8bc408b6
--- /dev/null
+++ b/plugins/dpdk/roles/ins_ovs/templates/start_ovsdb_server.sh.j2
@@ -0,0 +1,4 @@
+#!/bin/bash
+ovsdb-server --remote=punix:/usr/local/var/run/openvswitch/db.sock \
+          --remote=db:Open_vSwitch,Open_vSwitch,manager_options \
+          --pidfile --detach
diff --git a/plugins/dpdk/roles/ins_ovs/vars/main.yml b/plugins/dpdk/roles/ins_ovs/vars/main.yml
new file mode 100644
index 00000000..c5f14d4e
--- /dev/null
+++ b/plugins/dpdk/roles/ins_ovs/vars/main.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016-2017 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+dpdk_dir: /root/dpdk
+dpdk_build: '{{ dpdk_dir }}/x86_64-native-linuxapp-gcc'
+ovs_dir: /root/ovs
+ovs_repo: https://github.com/openvswitch/ovs.git
+ovs_version: v2.7.2