install pmu tools collectd plugin

Added review dependency, removed collectd commit-id
replicating bash script, added pmu roles to playbook

Create local mirror for event list generation, copy script
to image, and run event list generation at runtime if
intel_pmu is enabled.

Change-Id: Ie46a2b197f4d2037cf3eed194764ce9eb5670415
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

11 files changed, 325 insertions, 2 deletions

diff --git a/ansible/install_collectd.yml b/ansible/install_collectd.yml
index 647547053..0824c1525 100644
--- a/ansible/install_collectd.yml
+++ b/ansible/install_collectd.yml
@@ -23,6 +23,8 @@
     - install_rabbitmq
     - download_intel_cmt_cat
     - install_intel_cmt_cat
+    - download_pmu_tools
+    - install_pmu_tools
     - download_collectd
     - install_collectd
 
diff --git a/ansible/roles/download_pmu_tools/defaults/main.yml b/ansible/roles/download_pmu_tools/defaults/main.yml
new file mode 100644
index 000000000..90dae66cf
--- /dev/null
+++ b/ansible/roles/download_pmu_tools/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+pmu_tools_url: "https://github.com/andikleen/pmu-tools.git"
+pmu_tools_dest: "{{ clone_dest }}/pmu-tools"
+pmu_tools_version: "master"
+perfmon_url: "https://download.01.org/perfmon/"
diff --git a/ansible/roles/download_pmu_tools/files/event_download_local.py b/ansible/roles/download_pmu_tools/files/event_download_local.py
new file mode 100755
index 000000000..882fc5046
--- /dev/null
+++ b/ansible/roles/download_pmu_tools/files/event_download_local.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# Copyright (c) 2014, Intel Corporation
+# Author: Andi Kleen
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms and conditions of the GNU General Public License,
+# version 2, as published by the Free Software Foundation.
+#
+# This program is distributed in the hope it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# Automatic event list downloader
+#
+# event_download.py         download for current cpu
+# event_download.py -a      download all
+# event_download.py cpustr...  Download for specific CPU
+import sys
+import re
+from urllib2 import urlopen, URLError
+import os
+import string
+from fnmatch import fnmatch
+from shutil import copyfile
+
+urlpath = 'https://download.01.org/perfmon'
+localpath = 'download.01.org/perfmon'
+mapfile = 'mapfile.csv'
+modelpath = localpath + "/" + mapfile
+NSB_JSON = '/opt/nsb_bin/pmu_event.json'
+
+def get_cpustr():
+    f = open('/proc/cpuinfo', 'r')
+    cpu = [None, None, None]
+    for j in f:
+        n = j.split()
+        if n[0] == 'vendor_id':
+            cpu[0] = n[2]
+        elif n[0] == 'model' and n[1] == ':':
+            cpu[2] = int(n[2])
+        elif n[0] == 'cpu' and n[1] == 'family':
+            cpu[1] = int(n[3])
+        if all(cpu):
+            break
+    return "%s-%d-%X" % (cpu[0], cpu[1], cpu[2])
+
+def sanitize(s, a):
+    o = ""
+    for j in s:
+        if j in a:
+            o += j
+    return o
+
+def getdir():
+    try:
+        d = os.getenv("XDG_CACHE_HOME")
+        xd = d
+        if not d:
+            home = os.getenv("HOME")
+            d = "%s/.cache" % (home)
+        d += "/pmu-events"
+        if not os.path.isdir(d):
+            # try to handle the sudo case
+            if not xd:
+                user = os.getenv("SUDO_USER")
+                if user:
+                    nd = os.path.expanduser("~" + user) + "/.cache/pmu-events"
+                    if os.path.isdir(nd):
+                        return nd
+            os.makedirs(d)
+        return d
+    except OSError:
+        raise Exception('Cannot access ' + d)
+
+NUM_TRIES = 3
+
+def getfile(url, dir, fn):
+    tries = 0
+    print "Downloading", url, "to", fn
+    while True:
+        try:
+            f = open(url)
+            data = f.read()
+        except IOError:
+            tries += 1
+            if tries >= NUM_TRIES:
+                raise
+            print "retrying download"
+            continue
+        break
+    o = open(os.path.join(dir, fn), "w")
+    o.write(data)
+    o.close()
+    f.close()
+
+allowed_chars = string.ascii_letters + '_-.' + string.digits
+def download(match, key=None, link=True):
+    found = 0
+    dir = getdir()
+    try:
+        getfile(modelpath, dir, "mapfile.csv")
+        models = open(os.path.join(dir, "mapfile.csv"))
+        for j in models:
+            n = j.rstrip().split(",")
+            if len(n) < 4:
+                if len(n) > 0:
+                    print "Cannot parse", n
+                continue
+            cpu, version, name, type = n
+            if not fnmatch(cpu, match) or (key is not None and type not in key) or type.startswith("EventType"):
+                continue
+            cpu = sanitize(cpu, allowed_chars)
+            url = localpath + name
+	    fn = "%s-%s.json" % (cpu, sanitize(type, allowed_chars))
+            try:
+                os.remove(os.path.join(dir, fn))
+            except OSError:
+                pass
+            getfile(url, dir, fn)
+            if link:
+                lname = re.sub(r'.*/', '', name)
+                lname = sanitize(lname, allowed_chars)
+                try:
+                    os.remove(os.path.join(dir, lname))
+                except OSError:
+                    pass
+                try:
+                    os.symlink(fn, os.path.join(dir, lname))
+                except OSError as e:
+                    print >>sys.stderr, "Cannot link %s to %s:" % (name, lname), e
+            found += 1
+        models.close()
+        getfile(localpath + "/readme.txt", dir, "readme.txt")
+    except URLError as e:
+        print >>sys.stderr, "Cannot access event server:", e
+        print >>sys.stderr, "If you need a proxy to access the internet please set it with:"
+        print >>sys.stderr, "\texport https_proxy=http://proxyname..."
+        print >>sys.stderr, "If you are not connected to the internet please run this on a connected system:"
+        print >>sys.stderr, "\tevent_download.py '%s'" % (match)
+        print >>sys.stderr, "and then copy ~/.cache/pmu-events to the system under test"
+        print >>sys.stderr, "To get events for all possible CPUs use:"
+        print >>sys.stderr, "\tevent_download.py -a"
+    except OSError as e:
+        print >>sys.stderr, "Cannot write events file:", e
+    return found
+
+def download_current(link=False):
+    """Download JSON event list for current cpu.
+       Returns >0 when a event list is found"""
+    return download(get_cpustr(), link=link)
+
+def eventlist_name(name=None, key="core"):
+    if not name:
+        name = get_cpustr()
+    cache = getdir()
+    return "%s/%s-%s.json" % (cache, name, key)
+
+if __name__ == '__main__':
+    # only import argparse when actually called from command line
+    # this makes ocperf work on older python versions without it.
+    import argparse
+    p = argparse.ArgumentParser(usage='download Intel event files')
+    p.add_argument('--all', '-a', help='Download all available event files', action='store_true')
+    p.add_argument('--verbose', '-v', help='Be verbose', action='store_true')
+    p.add_argument('--mine', help='Print name of current CPU', action='store_true')
+    p.add_argument('--link', help='Create links with the original event file name', action='store_true', default=True)
+    p.add_argument('cpus', help='CPU identifiers to download', nargs='*')
+    args = p.parse_args()
+
+    cpustr = get_cpustr()
+    if args.verbose or args.mine:
+        print "My CPU", cpustr
+    if args.mine:
+        sys.exit(0)
+    d = getdir()
+    if args.all:
+	found = download('*', link=args.link)
+    elif len(args.cpus) == 0:
+        found = download_current(link=args.link)
+    else:
+        found = 0
+        for j in args.cpus:
+            found += download(j, link=args.link)
+
+    if found == 0:
+        print >>sys.stderr, "Nothing found"
+
+    el = eventlist_name()
+    if os.path.exists(el):
+        print "my event list", el 
+        copyfile(el,NSB_JSON)
+        print "File copied to ", NSB_JSON
diff --git a/ansible/roles/download_pmu_tools/tasks/main.yml b/ansible/roles/download_pmu_tools/tasks/main.yml
new file mode 100644
index 000000000..e78cc72d6
--- /dev/null
+++ b/ansible/roles/download_pmu_tools/tasks/main.yml
@@ -0,0 +1,41 @@
+# Copyright (c) 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+#- debug:
+#    var: pmu_tools_version
+#    verbosity: 2
+
+- name: fetch pmu_tools
+  git:
+    repo: "{{ pmu_tools_url }}"
+    dest: "{{ pmu_tools_dest }}"
+    version: "{{ pmu_tools_version }}"
+    accept_hostkey: yes
+    recursive: no
+    force: yes
+
+- set_fact:
+    pmu_tools_path: "{{ pmu_tools_dest }}"
+
+- name: Create perfmon local mirror
+  command: "wget -mkEpnp {{ perfmon_url }} -P {{ pmu_tools_dest }}"
+  ignore_errors: yes
+
+- name: Copy local event download file
+  copy:
+    src: event_download_local.py
+    dest: "{{ pmu_tools_dest }}"
+    owner: root
+    group: root
+    mode: 0755
diff --git a/ansible/roles/install_collectd/tasks/main.yml b/ansible/roles/install_collectd/tasks/main.yml
index 53f49097f..e3b655ffe 100644
--- a/ansible/roles/install_collectd/tasks/main.yml
+++ b/ansible/roles/install_collectd/tasks/main.yml
@@ -22,7 +22,7 @@
     chdir: "{{ collectd_path }}"
 
 - name: build collectd
-  shell: "./configure --with-libpqos=/usr/ --with-libdpdk={{ dpdk_shared_path }} --with-libyajl=/usr/ --enable-debug --enable-dpdkstat --enable-virt --enable-ovs_stats --prefix={{ INSTALL_BIN_PATH }}/collectd |& tee collectd-configure.txt"
+  shell: "./configure --with-libpqos=/usr/ --with-libdpdk={{ dpdk_shared_path }} --with-libyajl=/usr/ --with-libjevents=/usr/local --enable-debug --enable-dpdkstat --enable-virt --enable-ovs_stats --enable-intel_pmu --prefix={{ INSTALL_BIN_PATH }}/collectd |& tee collectd-configure.txt"
   args:
     executable: /bin/bash
     chdir: "{{ collectd_path }}"
diff --git a/ansible/roles/install_pmu_tools/tasks/main.yml b/ansible/roles/install_pmu_tools/tasks/main.yml
new file mode 100644
index 000000000..026fb67ea
--- /dev/null
+++ b/ansible/roles/install_pmu_tools/tasks/main.yml
@@ -0,0 +1,36 @@
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Install extra build dependencies
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items: "{{ pmu_tools_build_dependencies[ansible_os_family] }}"
+
+- name: fix pmu-tools CFLAGS to include -fPIC
+  replace:
+    path: "{{ pmu_tools_path }}/jevents/Makefile"
+    regexp: "CFLAGS := -g -Wall -O2 -Wno-unused-result"
+    replace: "CFLAGS := -g -Wall -O2 -Wno-unused-result -fPIC"
+    backup: yes
+
+- name: "make jevents"
+  make:
+    chdir: "{{ pmu_tools_path }}/jevents"
+
+- name: "make install jevents"
+  make:
+    chdir: "{{ pmu_tools_path }}/jevents"
+    target: install
+  become: yes
+
diff --git a/ansible/roles/install_pmu_tools/vars/main.yml b/ansible/roles/install_pmu_tools/vars/main.yml
new file mode 100644
index 000000000..10660f61b
--- /dev/null
+++ b/ansible/roles/install_pmu_tools/vars/main.yml
@@ -0,0 +1,38 @@
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+pmu_tools_build_dependencies:
+  Debian:
+    - flex
+    - bison
+    - build-essential
+    - pkg-config
+    - automake
+    - autotools-dev
+    - libltdl-dev
+    - librabbitmq-dev
+    - rabbitmq-server
+    - cmake
+    - libvirt-dev
+  RedHat:
+    - flex
+    - bison
+    - pkgconfig
+    - automake
+    - cmake
+    - rabbitmq-server
+    - librabbitmq-devel
+    - libtool-ltdl-devel
+    - libvirt-devel
+
diff --git a/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml b/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml
index c55886693..891ccd1dc 100644
--- a/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml
+++ b/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml
@@ -47,5 +47,7 @@
     - install_rabbitmq
     - download_intel_cmt_cat
     - install_intel_cmt_cat
+    - download_pmu_tools
+    - install_pmu_tools
     - download_collectd
     - install_collectd
diff --git a/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml b/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml
index af4b91a4e..79249dd37 100644
--- a/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml
+++ b/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml
@@ -56,5 +56,7 @@
     - install_rabbitmq
     - download_intel_cmt_cat
     - install_intel_cmt_cat
+    - download_pmu_tools
+    - install_pmu_tools
     - download_collectd
     - install_collectd
diff --git a/yardstick/network_services/nfvi/collectd.conf b/yardstick/network_services/nfvi/collectd.conf
index 22bd5d49d..e6a1f0d8c 100644
--- a/yardstick/network_services/nfvi/collectd.conf
+++ b/yardstick/network_services/nfvi/collectd.conf
@@ -86,7 +86,7 @@ LoadPlugin {{ plugin }}
    ReportHardwareCacheEvents true
    ReportKernelPMUEvents true
    ReportSoftwareEvents true
-   EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
+   EventList "/opt/nsb_bin/pmu_event.json"
    HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
 </Plugin>
 {% endif %}
diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py
index e3d0e3bca..a32948ae8 100644
--- a/yardstick/network_services/nfvi/resource.py
+++ b/yardstick/network_services/nfvi/resource.py
@@ -253,6 +253,10 @@ class ResourceProfile(object):
             # connection.execute("sudo %s '%s' '%s'" % (
             #     collectd_installer, http_proxy, https_proxy))
             return
+        if "intel_pmu" in self.plugins:
+            LOG.debug("Downloading event list for pmu_stats plugin")
+            cmd = 'sudo bash -c \'cd /opt/tempT/pmu-tools/; python event_download_local.py\''
+            connection.execute(cmd)
         LOG.debug("Starting collectd to collect NFVi stats")
         # ensure collectd.conf.d exists to avoid error/warning
         connection.execute("sudo mkdir -p /etc/collectd/collectd.conf.d")