diff options
| -rw-r--r-- | ansible/install_collectd.yml | 2 | ||||
| -rw-r--r-- | ansible/roles/download_pmu_tools/defaults/main.yml | 5 | ||||
| -rwxr-xr-x | ansible/roles/download_pmu_tools/files/event_download_local.py | 193 | ||||
| -rw-r--r-- | ansible/roles/download_pmu_tools/tasks/main.yml | 41 | ||||
| -rw-r--r-- | ansible/roles/install_collectd/tasks/main.yml | 2 | ||||
| -rw-r--r-- | ansible/roles/install_pmu_tools/tasks/main.yml | 36 | ||||
| -rw-r--r-- | ansible/roles/install_pmu_tools/vars/main.yml | 38 | ||||
| -rw-r--r-- | ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml | 2 | ||||
| -rw-r--r-- | ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml | 2 | ||||
| -rw-r--r-- | yardstick/network_services/nfvi/collectd.conf | 2 | ||||
| -rw-r--r-- | yardstick/network_services/nfvi/resource.py | 4 |
11 files changed, 325 insertions, 2 deletions
diff --git a/ansible/install_collectd.yml b/ansible/install_collectd.yml index 647547053..0824c1525 100644 --- a/ansible/install_collectd.yml +++ b/ansible/install_collectd.yml @@ -23,6 +23,8 @@ - install_rabbitmq - download_intel_cmt_cat - install_intel_cmt_cat + - download_pmu_tools + - install_pmu_tools - download_collectd - install_collectd diff --git a/ansible/roles/download_pmu_tools/defaults/main.yml b/ansible/roles/download_pmu_tools/defaults/main.yml new file mode 100644 index 000000000..90dae66cf --- /dev/null +++ b/ansible/roles/download_pmu_tools/defaults/main.yml @@ -0,0 +1,5 @@ +--- +pmu_tools_url: "https://github.com/andikleen/pmu-tools.git" +pmu_tools_dest: "{{ clone_dest }}/pmu-tools" +pmu_tools_version: "master" +perfmon_url: "https://download.01.org/perfmon/" diff --git a/ansible/roles/download_pmu_tools/files/event_download_local.py b/ansible/roles/download_pmu_tools/files/event_download_local.py new file mode 100755 index 000000000..882fc5046 --- /dev/null +++ b/ansible/roles/download_pmu_tools/files/event_download_local.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# Copyright (c) 2014, Intel Corporation +# Author: Andi Kleen +# +# This program is free software; you can redistribute it and/or modify it +# under the terms and conditions of the GNU General Public License, +# version 2, as published by the Free Software Foundation. +# +# This program is distributed in the hope it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# Automatic event list downloader +# +# event_download.py download for current cpu +# event_download.py -a download all +# event_download.py cpustr... Download for specific CPU +import sys +import re +from urllib2 import urlopen, URLError +import os +import string +from fnmatch import fnmatch +from shutil import copyfile + +urlpath = 'https://download.01.org/perfmon' +localpath = 'download.01.org/perfmon' +mapfile = 'mapfile.csv' +modelpath = localpath + "/" + mapfile +NSB_JSON = '/opt/nsb_bin/pmu_event.json' + +def get_cpustr(): + f = open('/proc/cpuinfo', 'r') + cpu = [None, None, None] + for j in f: + n = j.split() + if n[0] == 'vendor_id': + cpu[0] = n[2] + elif n[0] == 'model' and n[1] == ':': + cpu[2] = int(n[2]) + elif n[0] == 'cpu' and n[1] == 'family': + cpu[1] = int(n[3]) + if all(cpu): + break + return "%s-%d-%X" % (cpu[0], cpu[1], cpu[2]) + +def sanitize(s, a): + o = "" + for j in s: + if j in a: + o += j + return o + +def getdir(): + try: + d = os.getenv("XDG_CACHE_HOME") + xd = d + if not d: + home = os.getenv("HOME") + d = "%s/.cache" % (home) + d += "/pmu-events" + if not os.path.isdir(d): + # try to handle the sudo case + if not xd: + user = os.getenv("SUDO_USER") + if user: + nd = os.path.expanduser("~" + user) + "/.cache/pmu-events" + if os.path.isdir(nd): + return nd + os.makedirs(d) + return d + except OSError: + raise Exception('Cannot access ' + d) + +NUM_TRIES = 3 + +def getfile(url, dir, fn): + tries = 0 + print "Downloading", url, "to", fn + while True: + try: + f = open(url) + data = f.read() + except IOError: + tries += 1 + if tries >= NUM_TRIES: + raise + print "retrying download" + continue + break + o = open(os.path.join(dir, fn), "w") + o.write(data) + o.close() + f.close() + +allowed_chars = string.ascii_letters + '_-.' + string.digits +def download(match, key=None, link=True): + found = 0 + dir = getdir() + try: + getfile(modelpath, dir, "mapfile.csv") + models = open(os.path.join(dir, "mapfile.csv")) + for j in models: + n = j.rstrip().split(",") + if len(n) < 4: + if len(n) > 0: + print "Cannot parse", n + continue + cpu, version, name, type = n + if not fnmatch(cpu, match) or (key is not None and type not in key) or type.startswith("EventType"): + continue + cpu = sanitize(cpu, allowed_chars) + url = localpath + name + fn = "%s-%s.json" % (cpu, sanitize(type, allowed_chars)) + try: + os.remove(os.path.join(dir, fn)) + except OSError: + pass + getfile(url, dir, fn) + if link: + lname = re.sub(r'.*/', '', name) + lname = sanitize(lname, allowed_chars) + try: + os.remove(os.path.join(dir, lname)) + except OSError: + pass + try: + os.symlink(fn, os.path.join(dir, lname)) + except OSError as e: + print >>sys.stderr, "Cannot link %s to %s:" % (name, lname), e + found += 1 + models.close() + getfile(localpath + "/readme.txt", dir, "readme.txt") + except URLError as e: + print >>sys.stderr, "Cannot access event server:", e + print >>sys.stderr, "If you need a proxy to access the internet please set it with:" + print >>sys.stderr, "\texport https_proxy=http://proxyname..." + print >>sys.stderr, "If you are not connected to the internet please run this on a connected system:" + print >>sys.stderr, "\tevent_download.py '%s'" % (match) + print >>sys.stderr, "and then copy ~/.cache/pmu-events to the system under test" + print >>sys.stderr, "To get events for all possible CPUs use:" + print >>sys.stderr, "\tevent_download.py -a" + except OSError as e: + print >>sys.stderr, "Cannot write events file:", e + return found + +def download_current(link=False): + """Download JSON event list for current cpu. + Returns >0 when a event list is found""" + return download(get_cpustr(), link=link) + +def eventlist_name(name=None, key="core"): + if not name: + name = get_cpustr() + cache = getdir() + return "%s/%s-%s.json" % (cache, name, key) + +if __name__ == '__main__': + # only import argparse when actually called from command line + # this makes ocperf work on older python versions without it. + import argparse + p = argparse.ArgumentParser(usage='download Intel event files') + p.add_argument('--all', '-a', help='Download all available event files', action='store_true') + p.add_argument('--verbose', '-v', help='Be verbose', action='store_true') + p.add_argument('--mine', help='Print name of current CPU', action='store_true') + p.add_argument('--link', help='Create links with the original event file name', action='store_true', default=True) + p.add_argument('cpus', help='CPU identifiers to download', nargs='*') + args = p.parse_args() + + cpustr = get_cpustr() + if args.verbose or args.mine: + print "My CPU", cpustr + if args.mine: + sys.exit(0) + d = getdir() + if args.all: + found = download('*', link=args.link) + elif len(args.cpus) == 0: + found = download_current(link=args.link) + else: + found = 0 + for j in args.cpus: + found += download(j, link=args.link) + + if found == 0: + print >>sys.stderr, "Nothing found" + + el = eventlist_name() + if os.path.exists(el): + print "my event list", el + copyfile(el,NSB_JSON) + print "File copied to ", NSB_JSON diff --git a/ansible/roles/download_pmu_tools/tasks/main.yml b/ansible/roles/download_pmu_tools/tasks/main.yml new file mode 100644 index 000000000..e78cc72d6 --- /dev/null +++ b/ansible/roles/download_pmu_tools/tasks/main.yml @@ -0,0 +1,41 @@ +# Copyright (c) 2017 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +#- debug: +# var: pmu_tools_version +# verbosity: 2 + +- name: fetch pmu_tools + git: + repo: "{{ pmu_tools_url }}" + dest: "{{ pmu_tools_dest }}" + version: "{{ pmu_tools_version }}" + accept_hostkey: yes + recursive: no + force: yes + +- set_fact: + pmu_tools_path: "{{ pmu_tools_dest }}" + +- name: Create perfmon local mirror + command: "wget -mkEpnp {{ perfmon_url }} -P {{ pmu_tools_dest }}" + ignore_errors: yes + +- name: Copy local event download file + copy: + src: event_download_local.py + dest: "{{ pmu_tools_dest }}" + owner: root + group: root + mode: 0755 diff --git a/ansible/roles/install_collectd/tasks/main.yml b/ansible/roles/install_collectd/tasks/main.yml index 53f49097f..e3b655ffe 100644 --- a/ansible/roles/install_collectd/tasks/main.yml +++ b/ansible/roles/install_collectd/tasks/main.yml @@ -22,7 +22,7 @@ chdir: "{{ collectd_path }}" - name: build collectd - shell: "./configure --with-libpqos=/usr/ --with-libdpdk={{ dpdk_shared_path }} --with-libyajl=/usr/ --enable-debug --enable-dpdkstat --enable-virt --enable-ovs_stats --prefix={{ INSTALL_BIN_PATH }}/collectd |& tee collectd-configure.txt" + shell: "./configure --with-libpqos=/usr/ --with-libdpdk={{ dpdk_shared_path }} --with-libyajl=/usr/ --with-libjevents=/usr/local --enable-debug --enable-dpdkstat --enable-virt --enable-ovs_stats --enable-intel_pmu --prefix={{ INSTALL_BIN_PATH }}/collectd |& tee collectd-configure.txt" args: executable: /bin/bash chdir: "{{ collectd_path }}" diff --git a/ansible/roles/install_pmu_tools/tasks/main.yml b/ansible/roles/install_pmu_tools/tasks/main.yml new file mode 100644 index 000000000..026fb67ea --- /dev/null +++ b/ansible/roles/install_pmu_tools/tasks/main.yml @@ -0,0 +1,36 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +- name: Install extra build dependencies + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: "{{ pmu_tools_build_dependencies[ansible_os_family] }}" + +- name: fix pmu-tools CFLAGS to include -fPIC + replace: + path: "{{ pmu_tools_path }}/jevents/Makefile" + regexp: "CFLAGS := -g -Wall -O2 -Wno-unused-result" + replace: "CFLAGS := -g -Wall -O2 -Wno-unused-result -fPIC" + backup: yes + +- name: "make jevents" + make: + chdir: "{{ pmu_tools_path }}/jevents" + +- name: "make install jevents" + make: + chdir: "{{ pmu_tools_path }}/jevents" + target: install + become: yes + diff --git a/ansible/roles/install_pmu_tools/vars/main.yml b/ansible/roles/install_pmu_tools/vars/main.yml new file mode 100644 index 000000000..10660f61b --- /dev/null +++ b/ansible/roles/install_pmu_tools/vars/main.yml @@ -0,0 +1,38 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +pmu_tools_build_dependencies: + Debian: + - flex + - bison + - build-essential + - pkg-config + - automake + - autotools-dev + - libltdl-dev + - librabbitmq-dev + - rabbitmq-server + - cmake + - libvirt-dev + RedHat: + - flex + - bison + - pkgconfig + - automake + - cmake + - rabbitmq-server + - librabbitmq-devel + - libtool-ltdl-devel + - libvirt-devel + diff --git a/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml b/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml index c55886693..891ccd1dc 100644 --- a/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml +++ b/ansible/ubuntu_server_baremetal_deploy_samplevnfs.yml @@ -47,5 +47,7 @@ - install_rabbitmq - download_intel_cmt_cat - install_intel_cmt_cat + - download_pmu_tools + - install_pmu_tools - download_collectd - install_collectd diff --git a/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml b/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml index af4b91a4e..79249dd37 100644 --- a/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml +++ b/ansible/ubuntu_server_cloudimg_modify_samplevnfs.yml @@ -56,5 +56,7 @@ - install_rabbitmq - download_intel_cmt_cat - install_intel_cmt_cat + - download_pmu_tools + - install_pmu_tools - download_collectd - install_collectd diff --git a/yardstick/network_services/nfvi/collectd.conf b/yardstick/network_services/nfvi/collectd.conf index 22bd5d49d..e6a1f0d8c 100644 --- a/yardstick/network_services/nfvi/collectd.conf +++ b/yardstick/network_services/nfvi/collectd.conf @@ -86,7 +86,7 @@ LoadPlugin {{ plugin }} ReportHardwareCacheEvents true ReportKernelPMUEvents true ReportSoftwareEvents true - EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json" + EventList "/opt/nsb_bin/pmu_event.json" HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD" </Plugin> {% endif %} diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py index e3d0e3bca..a32948ae8 100644 --- a/yardstick/network_services/nfvi/resource.py +++ b/yardstick/network_services/nfvi/resource.py @@ -253,6 +253,10 @@ class ResourceProfile(object): # connection.execute("sudo %s '%s' '%s'" % ( # collectd_installer, http_proxy, https_proxy)) return + if "intel_pmu" in self.plugins: + LOG.debug("Downloading event list for pmu_stats plugin") + cmd = 'sudo bash -c \'cd /opt/tempT/pmu-tools/; python event_download_local.py\'' + connection.execute(cmd) LOG.debug("Starting collectd to collect NFVi stats") # ensure collectd.conf.d exists to avoid error/warning connection.execute("sudo mkdir -p /etc/collectd/collectd.conf.d") |