summaryrefslogtreecommitdiffstats
path: root/tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb')
-rw-r--r--tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb913
1 files changed, 913 insertions, 0 deletions
diff --git a/tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb b/tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb
new file mode 100644
index 00000000..0385b6f9
--- /dev/null
+++ b/tools/lma/metrics/jupyter-notebooks/Analysis-Monitoring-Local.ipynb
@@ -0,0 +1,913 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Metrics Analysis Notebook (local)\n",
+ "\n",
+ "#### Used to analyse / visualize the metrics when uploaded via csv file\n",
+ "\n",
+ "### Contributor: Aditya Srivastava <adityasrivastava301199@gmail.com>\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from datetime import datetime\n",
+ "import json\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib.dates as mdates\n",
+ "import numpy as np\n",
+ "import os\n",
+ "import pandas as pd\n",
+ "from pprint import pprint\n",
+ "import re\n",
+ "import requests\n",
+ "import time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Helper Functions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "DATETIME_FORMAT = \"%Y-%m-%d %H:%M:%S\"\n",
+ "\n",
+ "def convert_to_timestamp(s):\n",
+ " global DATETIME_FORMAT\n",
+ " return time.mktime(datetime.strptime(s, DATETIME_FORMAT).timetuple())\n",
+ "\n",
+ "def convert_to_time_string(epoch):\n",
+ " global DATETIME_FORMAT\n",
+ " t = datetime.fromtimestamp(float(epoch)/1000.)\n",
+ " return t.strftime(DATETIME_FORMAT)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Note: \n",
+ " \n",
+ "Path will be used as a parameter in almost every function\n",
+ "\n",
+ "path / rootdir / csv : (str) Path to the folder whose direct children are metric folders\n",
+ "\n",
+ "example: /path/to/folder\n",
+ "\n",
+ "When : \n",
+ "```sh\n",
+ "ls /path/to/folder\n",
+ "\n",
+ "# output should be directories such as\n",
+ "# cpu-0 cpu-1 cpu-2 ..........................\n",
+ "# processes-ovs-vswitchd ........processes-ovsdb-server\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Analysis Function"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### CPU"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_cpu_data(rootdir):\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\"cpu-\\d{1,2}\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " # read 3 files from this folder...\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames:\n",
+ " if 'user' in file:\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['user'] = temp_df['value']\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ "\n",
+ " if 'system' in file:\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['system'] = temp_df['value']\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ "\n",
+ " if 'idle' in file:\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['idle'] = temp_df['value']\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ "\n",
+ " _df['cpu'] = dirname.split('-')[-1]\n",
+ "\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ "\n",
+ " total = df['user'] + df['system'] + df['idle']\n",
+ "\n",
+ " df['user_percentage'] = df['user']*100 / total\n",
+ " df['system_percentage'] = df['system']*100 / total\n",
+ " df['idle_percentage'] = df['idle']*100 / total\n",
+ " \n",
+ " return df\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# CPU Unused Cores\n",
+ "def unused_cores(rootdir, verbose=False):\n",
+ " \n",
+ " df = fetch_cpu_data(rootdir)\n",
+ " groups = df.groupby(['cpu'])\n",
+ " if verbose: print(\"Unused Cores :\")\n",
+ "\n",
+ " unused_cores = []\n",
+ " for key, item in groups:\n",
+ " curr_df = item\n",
+ " unused_cores.append(key)\n",
+ " idle_values = curr_df.loc[curr_df['idle_percentage'] < 99.999]\n",
+ " if np.any(idle_values):\n",
+ " unused_cores.pop(-1)\n",
+ "\n",
+ " unused_cores = set(unused_cores)\n",
+ " for key, item in groups:\n",
+ " if key not in unused_cores:\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
+ "\n",
+ " ax1 = fig.add_subplot(131)\n",
+ " ax1.title.set_text(\"System\")\n",
+ " ax1.plot(item['epoch'], item['system_percentage'])\n",
+ " \n",
+ " ax2 = fig.add_subplot(132)\n",
+ " ax2.title.set_text(\"User\")\n",
+ " ax2.plot(item['epoch'], item['user_percentage'])\n",
+ " \n",
+ " ax3 = fig.add_subplot(133)\n",
+ " ax3.title.set_text(\"Idle\")\n",
+ " ax3.plot(item['epoch'], item['idle_percentage'])\n",
+ "\n",
+ " plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
+ " plt.show()\n",
+ "\n",
+ " print(\"Number of unused cores: \", len(unused_cores))\n",
+ " return unused_cores\n",
+ "\n",
+ "\n",
+ "#CPU fully used cores\n",
+ "def fully_used_cores(rootdir, verbose=False):\n",
+ " \n",
+ "\n",
+ " df = fetch_cpu_data(rootdir)\n",
+ " groups = df.groupby(['cpu'])\n",
+ " if verbose: print(\"Fully Used Cores :\")\n",
+ "\n",
+ " fully_used_cores = []\n",
+ " for key, item in groups:\n",
+ " curr_df = item\n",
+ " idle_values = curr_df.loc[curr_df['idle_percentage'] <= 10]\n",
+ " if np.any(idle_values):\n",
+ " fully_used_cores.append(key)\n",
+ "\n",
+ " fully_used_cores = set(fully_used_cores)\n",
+ " for key, item in groups:\n",
+ " if key not in fully_used_cores:\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
+ "\n",
+ " ax1 = fig.add_subplot(131)\n",
+ " ax1.title.set_text(\"System\")\n",
+ " ax1.plot(item['epoch'], item['system_percentage'])\n",
+ "\n",
+ " ax2 = fig.add_subplot(132)\n",
+ " ax2.title.set_text(\"User\")\n",
+ " ax2.plot(item['epoch'], item['user_percentage'])\n",
+ "\n",
+ " ax3 = fig.add_subplot(133)\n",
+ " ax3.title.set_text(\"Idle\")\n",
+ " ax3.plot(item['epoch'], item['idle_percentage'])\n",
+ "\n",
+ " plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
+ " plt.show()\n",
+ "\n",
+ " print(\"Number of fully used cores: \", len(fully_used_cores))\n",
+ " return fully_used_cores\n",
+ "\n",
+ "\n",
+ "# CPU used cores plots\n",
+ "def used_cores(rootdir, verbose=False):\n",
+ "\n",
+ " df = fetch_cpu_data(rootdir)\n",
+ " groups = df.groupby(['cpu'])\n",
+ " if verbose: print(\"Used Cores :\")\n",
+ "\n",
+ " used_cores = []\n",
+ " for key, item in groups:\n",
+ " curr_df = item\n",
+ " idle_values = curr_df.loc[curr_df['idle_percentage'] < 99.999]\n",
+ " if np.any(idle_values):\n",
+ " used_cores.append(key)\n",
+ "\n",
+ " used_cores = set(used_cores)\n",
+ " for key, item in groups:\n",
+ " if key not in used_cores:\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
+ "\n",
+ " ax1 = fig.add_subplot(131)\n",
+ " ax1.title.set_text(\"System\")\n",
+ " ax1.plot(item['epoch'], item['system_percentage'])\n",
+ "\n",
+ " ax2 = fig.add_subplot(132)\n",
+ " ax2.title.set_text(\"User\")\n",
+ " ax2.plot(item['epoch'], item['user_percentage'])\n",
+ "\n",
+ " ax3 = fig.add_subplot(133)\n",
+ " ax3.title.set_text(\"Idle\")\n",
+ " ax3.plot(item['epoch'], item['idle_percentage'])\n",
+ "\n",
+ " plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
+ " plt.show()\n",
+ "\n",
+ " print(\"Number of used cores: \", len(used_cores))\n",
+ " return used_cores\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Interface"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_interfaces_data(rootdir):\n",
+ "\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\"interface-.*\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " # read 3 files from this folder...\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames:\n",
+ " if 'errors' in file:\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['error_rx'] = temp_df['rx']\n",
+ " _df['error_tx'] = temp_df['tx']\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ "\n",
+ " if 'dropped' in file:\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['dropped_rx'] = temp_df['rx']\n",
+ " _df['dropped_tx'] = temp_df['tx']\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ "\n",
+ " _df['interface'] = '-'.join(dirname.split('-')[1:])\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ " return df\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Interface Dropped (both type 1 and 2, i.e rx and tx)\n",
+ "def interface_dropped(rootdir, verbose=False):\n",
+ " \n",
+ " df = fetch_interfaces_data(rootdir)\n",
+ " group = df.groupby(['interface'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " dropped = {'rx':[], 'tx':[]}\n",
+ "\n",
+ " itr = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ "\n",
+ " if np.any(curr_df['dropped_rx'] == 1):\n",
+ " dropped_rows = curr_df[curr_df['dropped_rx'] == 1]\n",
+ " dropped['rx'].append([key, dropped_row['epoch'].iloc[0]])\n",
+ " if np.any(curr_df['dropped_tx'] == 1):\n",
+ " dropped_rows = curr_df[curr_df['dropped_tx'] == 1]\n",
+ " dropped['tx'].append([key, dropped_row['epoch'].iloc[0]])\n",
+ "\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[itr%2], edgecolor='red')\n",
+ " ax = fig.add_subplot(211)\n",
+ " ax.title.set_text(\"Interface: {} Dropped (rx)\".format(key))\n",
+ " ax.plot(item['epoch'], item['dropped_rx'])\n",
+ "\n",
+ " ax1 = fig.add_subplot(212)\n",
+ " ax1.title.set_text(\"Interface: {} Dropped (tx)\".format(key))\n",
+ " ax1.plot(item['epoch'], item['dropped_tx'])\n",
+ "\n",
+ " itr += 1\n",
+ "\n",
+ " plt.suptitle('Interface Dropped', fontsize=14)\n",
+ " plt.show()\n",
+ "\n",
+ " return dropped\n",
+ "\n",
+ "\n",
+ "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
+ "def interface_errors(rootdir, verbose=False):\n",
+ " \n",
+ " df = fetch_interfaces_data(rootdir)\n",
+ " group = df.groupby(['interface'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " errors = {'rx':[], 'tx':[]}\n",
+ "\n",
+ " itr = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ "\n",
+ " if np.any(curr_df['error_rx'] == 1):\n",
+ " err_rows = curr_df[curr_df['error_rx'] == 1]\n",
+ " errors['rx'].append([key, err_row['epoch'].iloc[0]])\n",
+ " if np.any(curr_df['error_tx'] == 1):\n",
+ " err_rows = curr_df[curr_df['error_tx'] == 1]\n",
+ " errors['tx'].append([key, err_row['epoch'].iloc[0]])\n",
+ "\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[itr%2], edgecolor='red')\n",
+ " ax = fig.add_subplot(211)\n",
+ " ax.title.set_text(\"Interface: {} Errors (rx)\".format(key))\n",
+ " ax.plot(item['epoch'], item['error_rx'])\n",
+ "\n",
+ " ax1 = fig.add_subplot(212)\n",
+ " ax1.title.set_text(\"Interface: {} Errors (tx)\".format(key))\n",
+ " ax1.plot(item['epoch'], item['error_tx'])\n",
+ "\n",
+ " itr += 1\n",
+ "\n",
+ " plt.suptitle('Interface Erros', fontsize=14)\n",
+ " plt.show()\n",
+ "\n",
+ " return errors\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### OVS Stats (Non DPDK)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_ovs_stats_data(rootdir):\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\"ovs_stats-.*\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " if 'dpdk' in dirname:\n",
+ " continue #ignoring dpdk\n",
+ "\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames:\n",
+ " if 'errors' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ "\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df\n",
+ "\n",
+ " if 'dropped' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df \n",
+ " _df['interface'] = '-'.join(dirname.split('-')[1:])\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ " return df\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def ovs_stats_dropped(rootdir, verbose=False):\n",
+ " \n",
+ " df = fetch_ovs_stats_data(rootdir)\n",
+ " group = df.groupby(['interface'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'dropped' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"Interface: {} Dropped {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()\n",
+ " return\n",
+ "\n",
+ "\n",
+ "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
+ "def ovs_stats_errors(rootdir, verbose=False):\n",
+ "\n",
+ "\n",
+ " df = fetch_ovs_stats_data(rootdir)\n",
+ " group = df.groupby(['interface'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'error' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"Interface: {} Errors {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### DPDK"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_dpdk_data(rootdir):\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\".*dpdk.*\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames:\n",
+ " if 'errors' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ "\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df\n",
+ "\n",
+ " if 'dropped' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df \n",
+ " _df['dpdk'] = '-'.join(dirname.split('-')[1:])\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ " return df\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fetch_dpdk_data(rootdir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def dpdk_dropped(rootdir, verbose=False):\n",
+ " \n",
+ " df = fetch_dpdk_data(rootdir)\n",
+ " group = df.groupby(['dpdk'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'dropped' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"DpDK: {} Dropped {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()\n",
+ " return\n",
+ "\n",
+ "\n",
+ "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
+ "def dpdk_errors(rootdir, verbose=False):\n",
+ "\n",
+ "\n",
+ " df = fetch_dpdk_data(rootdir)\n",
+ " group = df.groupby(['dpdk'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'error' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"DpDK: {} Errors {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dpdk_dropped(rootdir)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### RDT (need to be testes)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_rdt_data(rootdir):\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\".*rdt.*\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames:\n",
+ " if 'bytes' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ "\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df\n",
+ " \n",
+ " if 'bandwidth' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ "\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df\n",
+ "\n",
+ " if 'ipc' in file:\n",
+ " col_name = '-'.join(file.split('_')[1:])\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df \n",
+ " _df['intel_rdt'] = '-'.join(dirname.split('-')[1:])\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ " return df\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# L3 cache bytes\n",
+ "def plot_rdt_bytes(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
+ " \n",
+ " df = fetch_rdt_data(rootdir)\n",
+ " group = df.groupby(['intel_rdt'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'bytes' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"RDT BYTES, RDT: {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()\n",
+ "\n",
+ "\n",
+ "# L3 IPC values\n",
+ "def plot_rdt_ipc(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
+ " \n",
+ " \n",
+ " df = fetch_rdt_data(rootdir)\n",
+ " group = df.groupby(['intel_rdt'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'ipc' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"RDT IPC, RDT: {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()\n",
+ "\n",
+ "\n",
+ "\n",
+ "# memeory bandwidtdh\n",
+ "def get_rdt_memory_bandwidth(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
+ " \n",
+ " \n",
+ " df = fetch_rdt_data(rootdir)\n",
+ " group = df.groupby(['intel_rdt'])\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ "\n",
+ " i = 0\n",
+ " for key, item in group:\n",
+ " curr_df = item\n",
+ " for col in curr_df:\n",
+ " if 'bandwidht' in col:\n",
+ " if item[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(item['epoch'], item[col])\n",
+ " plt.title(\"RDT Memory Bandwidht, RDT: {}\".format(key, col))\n",
+ " i += 1\n",
+ " plt.show()\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Memory (following functions still need to written for csv)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rootdir = 'metrics_data/'\n",
+ "\n",
+ "def fetch_memory_data(rootdir):\n",
+ " df = pd.DataFrame()\n",
+ " reg_compile = re.compile(\"memory\")\n",
+ " for dirpath, dirnames, filenames in os.walk(rootdir):\n",
+ " dirname = dirpath.split(os.sep)[-1] \n",
+ " if reg_compile.match(dirname):\n",
+ " print(dirname)\n",
+ " _df = pd.DataFrame()\n",
+ " for file in filenames: \n",
+ " col_name = file.split('-')[1]\n",
+ " temp_df = pd.read_csv(dirpath + os.sep + file)\n",
+ " _df['epoch'] = temp_df['epoch']\n",
+ " temp_df = temp_df.drop(['epoch'], axis=1)\n",
+ " new_cols = [col_name for i in temp_df.columns]\n",
+ " _df[new_cols] = temp_df\n",
+ " df = df.append(_df, ignore_index=True)\n",
+ " return df"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "def get_memory_usage(rootdir, verbose=False):\n",
+ " df = fetch_memory_data(rootdir)\n",
+ " color = ['oldlace', 'mistyrose']\n",
+ " i = 0\n",
+ " for col in df:\n",
+ " if df[col].isnull().all():\n",
+ " continue\n",
+ " fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
+ " plt.plot(df['epoch'], df[col])\n",
+ " plt.title(\"{} Memory\".format(col))\n",
+ " i += 1\n",
+ " plt.show()\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ " "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Usage / Examples\n",
+ "\n",
+ "\n",
+ "##### CPU \n",
+ "\n",
+ "- For calling cpu unsued cores\n",
+ "\n",
+ "```py\n",
+ "cores = unused_cores(rootdir='metrics_data')\n",
+ "```\n",
+ "\n",
+ "- For finding fully used cores\n",
+ "\n",
+ "```py\n",
+ "fully_used = fully_used_cores('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- Similarly for plotting used cores\n",
+ "\n",
+ "```py\n",
+ "plot_used_cores(csv='metrics_data')\n",
+ "```\n",
+ "\n",
+ "\n",
+ "##### Interface\n",
+ "\n",
+ "- Interface Dropped \n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "dropped_interfaces = interface_dropped('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- Interface Errors\n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "interface_errors('metrics_data')\n",
+ "```\n",
+ "\n",
+ "##### OVS Stats\n",
+ "\n",
+ "- OVS Stats Dropped \n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "ovs_stats_dropped('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- OVS Stats Errors\n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "ovs_stats_errors('metrics_data')\n",
+ "```\n",
+ "\n",
+ "##### DPDK \n",
+ "\n",
+ "- DPDK Dropped \n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "dpdk_dropped('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- DPDK Errors\n",
+ "\n",
+ "```py\n",
+ "# Using CSV\n",
+ "dpdk_errors('metrics_data')\n",
+ "```\n",
+ "\n",
+ "\n",
+ "\n",
+ "##### RDT (Do not run yet)\n",
+ "\n",
+ "- Plot bytes\n",
+ "\n",
+ "```py\n",
+ "#csv\n",
+ "plot_rdt_bytes('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- Plot ipc values\n",
+ "\n",
+ "```py\n",
+ "#csv\n",
+ "plot_rdt_ipc('metrics_data')\n",
+ "```\n",
+ "\n",
+ "- Memory bandwidth\n",
+ "\n",
+ "```py\n",
+ "#csv\n",
+ "get_rdt_memory_bandwidth('metrics_data')\n",
+ "```\n",
+ "\n",
+ "##### Memory\n",
+ "\n",
+ "```py\n",
+ "#csv\n",
+ "get_memory_usage('metrics_data')\n",
+ "```"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.8"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}