{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Metrics Analysis Notebook (local)\n",
    "\n",
    "#### Used to analyse / visualize the metrics when uploaded via csv file\n",
    "\n",
    "### Contributor:    Aditya Srivastava <adityasrivastava301199@gmail.com>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.dates as mdates\n",
    "import numpy as np\n",
    "import os\n",
    "import pandas as pd\n",
    "from pprint import pprint\n",
    "import re\n",
    "import requests\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATETIME_FORMAT = \"%Y-%m-%d %H:%M:%S\"\n",
    "\n",
    "def convert_to_timestamp(s):\n",
    "    global DATETIME_FORMAT\n",
    "    return time.mktime(datetime.strptime(s, DATETIME_FORMAT).timetuple())\n",
    "\n",
    "def convert_to_time_string(epoch):\n",
    "    global DATETIME_FORMAT\n",
    "    t = datetime.fromtimestamp(float(epoch)/1000.)\n",
    "    return t.strftime(DATETIME_FORMAT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Note: \n",
    "    \n",
    "Path will be used as a parameter in almost every function\n",
    "\n",
    "path / rootdir / csv : (str) Path to the folder whose direct children are metric folders\n",
    "\n",
    "example: /path/to/folder\n",
    "\n",
    "When : \n",
    "```sh\n",
    "ls /path/to/folder\n",
    "\n",
    "# output should be directories such as\n",
    "# cpu-0 cpu-1 cpu-2  ..........................\n",
    "# processes-ovs-vswitchd ........processes-ovsdb-server\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analysis Function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_cpu_data(rootdir):\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\"cpu-\\d{1,2}\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            # read 3 files from this folder...\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:\n",
    "                if 'user' in file:\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['user'] = temp_df['value']\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "\n",
    "                if 'system' in file:\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['system'] = temp_df['value']\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "\n",
    "                if 'idle' in file:\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['idle'] = temp_df['value']\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "\n",
    "            _df['cpu'] = dirname.split('-')[-1]\n",
    "\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "\n",
    "    total = df['user'] + df['system'] + df['idle']\n",
    "\n",
    "    df['user_percentage'] = df['user']*100 / total\n",
    "    df['system_percentage'] = df['system']*100 / total\n",
    "    df['idle_percentage'] = df['idle']*100 / total\n",
    "    \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CPU Unused Cores\n",
    "def unused_cores(rootdir, verbose=False):\n",
    "    \n",
    "    df = fetch_cpu_data(rootdir)\n",
    "    groups = df.groupby(['cpu'])\n",
    "    if verbose: print(\"Unused Cores :\")\n",
    "\n",
    "    unused_cores = []\n",
    "    for key, item in groups:\n",
    "        curr_df = item\n",
    "        unused_cores.append(key)\n",
    "        idle_values = curr_df.loc[curr_df['idle_percentage'] < 99.999]\n",
    "        if np.any(idle_values):\n",
    "            unused_cores.pop(-1)\n",
    "\n",
    "    unused_cores = set(unused_cores)\n",
    "    for key, item in groups:\n",
    "        if key not in unused_cores:\n",
    "            continue\n",
    "        fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
    "\n",
    "        ax1 = fig.add_subplot(131)\n",
    "        ax1.title.set_text(\"System\")\n",
    "        ax1.plot(item['epoch'], item['system_percentage'])\n",
    "    \n",
    "        ax2 = fig.add_subplot(132)\n",
    "        ax2.title.set_text(\"User\")\n",
    "        ax2.plot(item['epoch'], item['user_percentage'])\n",
    "            \n",
    "        ax3 = fig.add_subplot(133)\n",
    "        ax3.title.set_text(\"Idle\")\n",
    "        ax3.plot(item['epoch'], item['idle_percentage'])\n",
    "\n",
    "        plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
    "        plt.show()\n",
    "\n",
    "    print(\"Number of unused cores:   \", len(unused_cores))\n",
    "    return unused_cores\n",
    "\n",
    "\n",
    "#CPU fully used cores\n",
    "def fully_used_cores(rootdir, verbose=False):\n",
    "    \n",
    "\n",
    "    df = fetch_cpu_data(rootdir)\n",
    "    groups = df.groupby(['cpu'])\n",
    "    if verbose: print(\"Fully Used Cores :\")\n",
    "\n",
    "    fully_used_cores = []\n",
    "    for key, item in groups:\n",
    "        curr_df = item\n",
    "        idle_values = curr_df.loc[curr_df['idle_percentage'] <= 10]\n",
    "        if np.any(idle_values):\n",
    "            fully_used_cores.append(key)\n",
    "\n",
    "    fully_used_cores = set(fully_used_cores)\n",
    "    for key, item in groups:\n",
    "        if key not in fully_used_cores:\n",
    "            continue\n",
    "        fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
    "\n",
    "        ax1 = fig.add_subplot(131)\n",
    "        ax1.title.set_text(\"System\")\n",
    "        ax1.plot(item['epoch'], item['system_percentage'])\n",
    "\n",
    "        ax2 = fig.add_subplot(132)\n",
    "        ax2.title.set_text(\"User\")\n",
    "        ax2.plot(item['epoch'], item['user_percentage'])\n",
    "\n",
    "        ax3 = fig.add_subplot(133)\n",
    "        ax3.title.set_text(\"Idle\")\n",
    "        ax3.plot(item['epoch'], item['idle_percentage'])\n",
    "\n",
    "        plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
    "        plt.show()\n",
    "\n",
    "    print(\"Number of fully used cores:   \", len(fully_used_cores))\n",
    "    return fully_used_cores\n",
    "\n",
    "\n",
    "# CPU used cores plots\n",
    "def used_cores(rootdir, verbose=False):\n",
    "\n",
    "    df = fetch_cpu_data(rootdir)\n",
    "    groups = df.groupby(['cpu'])\n",
    "    if verbose: print(\"Used Cores :\")\n",
    "\n",
    "    used_cores = []\n",
    "    for key, item in groups:\n",
    "        curr_df = item\n",
    "        idle_values = curr_df.loc[curr_df['idle_percentage'] < 99.999]\n",
    "        if np.any(idle_values):\n",
    "            used_cores.append(key)\n",
    "\n",
    "    used_cores = set(used_cores)\n",
    "    for key, item in groups:\n",
    "        if key not in used_cores:\n",
    "            continue\n",
    "        fig = plt.figure(figsize=(24,6), facecolor='oldlace', edgecolor='red')\n",
    "\n",
    "        ax1 = fig.add_subplot(131)\n",
    "        ax1.title.set_text(\"System\")\n",
    "        ax1.plot(item['epoch'], item['system_percentage'])\n",
    "\n",
    "        ax2 = fig.add_subplot(132)\n",
    "        ax2.title.set_text(\"User\")\n",
    "        ax2.plot(item['epoch'], item['user_percentage'])\n",
    "\n",
    "        ax3 = fig.add_subplot(133)\n",
    "        ax3.title.set_text(\"Idle\")\n",
    "        ax3.plot(item['epoch'], item['idle_percentage'])\n",
    "\n",
    "        plt.suptitle('Used CPU Core {}'.format(key), fontsize=14)\n",
    "        plt.show()\n",
    "\n",
    "    print(\"Number of used cores:   \", len(used_cores))\n",
    "    return used_cores\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Interface"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_interfaces_data(rootdir):\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\"interface-.*\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            # read 3 files from this folder...\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:\n",
    "                if 'errors' in file:\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['error_rx'] = temp_df['rx']\n",
    "                    _df['error_tx'] = temp_df['tx']\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "\n",
    "                if 'dropped' in file:\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['dropped_rx'] = temp_df['rx']\n",
    "                    _df['dropped_tx'] = temp_df['tx']\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "\n",
    "            _df['interface'] = '-'.join(dirname.split('-')[1:])\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interface Dropped (both type 1 and 2, i.e rx and tx)\n",
    "def interface_dropped(rootdir, verbose=False):\n",
    "        \n",
    "    df = fetch_interfaces_data(rootdir)\n",
    "    group = df.groupby(['interface'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    dropped = {'rx':[], 'tx':[]}\n",
    "\n",
    "    itr = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "\n",
    "        if np.any(curr_df['dropped_rx'] == 1):\n",
    "            dropped_rows = curr_df[curr_df['dropped_rx'] == 1]\n",
    "            dropped['rx'].append([key, dropped_row['epoch'].iloc[0]])\n",
    "        if np.any(curr_df['dropped_tx'] == 1):\n",
    "            dropped_rows = curr_df[curr_df['dropped_tx'] == 1]\n",
    "            dropped['tx'].append([key, dropped_row['epoch'].iloc[0]])\n",
    "\n",
    "        fig = plt.figure(figsize=(24,6), facecolor=color[itr%2], edgecolor='red')\n",
    "        ax = fig.add_subplot(211)\n",
    "        ax.title.set_text(\"Interface: {} Dropped (rx)\".format(key))\n",
    "        ax.plot(item['epoch'], item['dropped_rx'])\n",
    "\n",
    "        ax1 = fig.add_subplot(212)\n",
    "        ax1.title.set_text(\"Interface: {} Dropped (tx)\".format(key))\n",
    "        ax1.plot(item['epoch'], item['dropped_tx'])\n",
    "\n",
    "        itr += 1\n",
    "\n",
    "    plt.suptitle('Interface Dropped', fontsize=14)\n",
    "    plt.show()\n",
    "\n",
    "    return dropped\n",
    "\n",
    "\n",
    "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
    "def interface_errors(rootdir, verbose=False):\n",
    "        \n",
    "    df = fetch_interfaces_data(rootdir)\n",
    "    group = df.groupby(['interface'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    errors = {'rx':[], 'tx':[]}\n",
    "\n",
    "    itr = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "\n",
    "        if np.any(curr_df['error_rx'] == 1):\n",
    "            err_rows = curr_df[curr_df['error_rx'] == 1]\n",
    "            errors['rx'].append([key, err_row['epoch'].iloc[0]])\n",
    "        if np.any(curr_df['error_tx'] == 1):\n",
    "            err_rows = curr_df[curr_df['error_tx'] == 1]\n",
    "            errors['tx'].append([key, err_row['epoch'].iloc[0]])\n",
    "\n",
    "        fig = plt.figure(figsize=(24,6), facecolor=color[itr%2], edgecolor='red')\n",
    "        ax = fig.add_subplot(211)\n",
    "        ax.title.set_text(\"Interface: {} Errors (rx)\".format(key))\n",
    "        ax.plot(item['epoch'], item['error_rx'])\n",
    "\n",
    "        ax1 = fig.add_subplot(212)\n",
    "        ax1.title.set_text(\"Interface: {} Errors (tx)\".format(key))\n",
    "        ax1.plot(item['epoch'], item['error_tx'])\n",
    "\n",
    "        itr += 1\n",
    "\n",
    "    plt.suptitle('Interface Erros', fontsize=14)\n",
    "    plt.show()\n",
    "\n",
    "    return errors\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OVS Stats (Non DPDK)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_ovs_stats_data(rootdir):\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\"ovs_stats-.*\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            if 'dpdk' in dirname:\n",
    "                continue #ignoring dpdk\n",
    "\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:\n",
    "                if 'errors' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df\n",
    "\n",
    "                if 'dropped' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df            \n",
    "            _df['interface'] = '-'.join(dirname.split('-')[1:])\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ovs_stats_dropped(rootdir, verbose=False):\n",
    "    \n",
    "    df = fetch_ovs_stats_data(rootdir)\n",
    "    group = df.groupby(['interface'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'dropped' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"Interface: {} Dropped {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()\n",
    "    return\n",
    "\n",
    "\n",
    "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
    "def ovs_stats_errors(rootdir, verbose=False):\n",
    "\n",
    "\n",
    "    df = fetch_ovs_stats_data(rootdir)\n",
    "    group = df.groupby(['interface'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'error' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"Interface: {} Errors {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DPDK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_dpdk_data(rootdir):\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\".*dpdk.*\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:\n",
    "                if 'errors' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df\n",
    "\n",
    "                if 'dropped' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df            \n",
    "            _df['dpdk'] = '-'.join(dirname.split('-')[1:])\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_dpdk_data(rootdir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dpdk_dropped(rootdir, verbose=False):\n",
    "    \n",
    "    df = fetch_dpdk_data(rootdir)\n",
    "    group = df.groupby(['dpdk'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'dropped' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"DpDK: {} Dropped {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()\n",
    "    return\n",
    "\n",
    "\n",
    "# Interface Errors (both type 1 and 2, i.e rx and tx)\n",
    "def dpdk_errors(rootdir, verbose=False):\n",
    "\n",
    "\n",
    "    df = fetch_dpdk_data(rootdir)\n",
    "    group = df.groupby(['dpdk'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'error' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"DpDK: {} Errors {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpdk_dropped(rootdir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RDT  (need to be testes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_rdt_data(rootdir):\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\".*rdt.*\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:\n",
    "                if 'bytes' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df\n",
    "                    \n",
    "                if 'bandwidth' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df\n",
    "\n",
    "                if 'ipc' in file:\n",
    "                    col_name = '-'.join(file.split('_')[1:])\n",
    "                    temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                    _df['epoch'] = temp_df['epoch']\n",
    "                    temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                    new_cols = [i + '_' + col_name for i in temp_df.columns]\n",
    "                    _df[new_cols] = temp_df            \n",
    "            _df['intel_rdt'] = '-'.join(dirname.split('-')[1:])\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# L3 cache bytes\n",
    "def plot_rdt_bytes(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
    "    \n",
    "    df = fetch_rdt_data(rootdir)\n",
    "    group = df.groupby(['intel_rdt'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'bytes' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"RDT BYTES, RDT: {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "# L3 IPC values\n",
    "def plot_rdt_ipc(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
    "    \n",
    "    \n",
    "    df = fetch_rdt_data(rootdir)\n",
    "    group = df.groupby(['intel_rdt'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'ipc' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"RDT IPC, RDT: {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "\n",
    "# memeory bandwidtdh\n",
    "def get_rdt_memory_bandwidth(start=None, end=None, node=None, steps='15s', csv=None, verbose=False):\n",
    "    \n",
    "        \n",
    "    df = fetch_rdt_data(rootdir)\n",
    "    group = df.groupby(['intel_rdt'])\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "\n",
    "    i = 0\n",
    "    for key, item in group:\n",
    "        curr_df = item\n",
    "        for col in curr_df:\n",
    "            if 'bandwidht' in col:\n",
    "                if item[col].isnull().all():\n",
    "                    continue\n",
    "                fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "                plt.plot(item['epoch'], item[col])\n",
    "                plt.title(\"RDT Memory Bandwidht, RDT: {}\".format(key, col))\n",
    "        i += 1\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Memory (following functions still need to written for csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rootdir = 'metrics_data/'\n",
    "\n",
    "def fetch_memory_data(rootdir):\n",
    "    df = pd.DataFrame()\n",
    "    reg_compile = re.compile(\"memory\")\n",
    "    for dirpath, dirnames, filenames in os.walk(rootdir):\n",
    "        dirname = dirpath.split(os.sep)[-1] \n",
    "        if reg_compile.match(dirname):\n",
    "            print(dirname)\n",
    "            _df = pd.DataFrame()\n",
    "            for file in filenames:                \n",
    "                col_name = file.split('-')[1]\n",
    "                temp_df = pd.read_csv(dirpath + os.sep + file)\n",
    "                _df['epoch'] = temp_df['epoch']\n",
    "                temp_df = temp_df.drop(['epoch'], axis=1)\n",
    "                new_cols = [col_name for i in temp_df.columns]\n",
    "                _df[new_cols] = temp_df\n",
    "            df = df.append(_df, ignore_index=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_memory_usage(rootdir, verbose=False):\n",
    "    df = fetch_memory_data(rootdir)\n",
    "    color = ['oldlace', 'mistyrose']\n",
    "    i = 0\n",
    "    for col in df:\n",
    "        if df[col].isnull().all():\n",
    "            continue\n",
    "        fig = plt.figure(figsize=(24,6), facecolor=color[i%2], edgecolor='red')\n",
    "        plt.plot(df['epoch'], df[col])\n",
    "        plt.title(\"{} Memory\".format(col))\n",
    "        i += 1\n",
    "    plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Usage / Examples\n",
    "\n",
    "\n",
    "##### CPU \n",
    "\n",
    "- For calling cpu unsued cores\n",
    "\n",
    "```py\n",
    "cores = unused_cores(rootdir='metrics_data')\n",
    "```\n",
    "\n",
    "- For finding fully used cores\n",
    "\n",
    "```py\n",
    "fully_used = fully_used_cores('metrics_data')\n",
    "```\n",
    "\n",
    "- Similarly for plotting used cores\n",
    "\n",
    "```py\n",
    "plot_used_cores(csv='metrics_data')\n",
    "```\n",
    "\n",
    "\n",
    "##### Interface\n",
    "\n",
    "- Interface Dropped  \n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "dropped_interfaces = interface_dropped('metrics_data')\n",
    "```\n",
    "\n",
    "- Interface Errors\n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "interface_errors('metrics_data')\n",
    "```\n",
    "\n",
    "##### OVS Stats\n",
    "\n",
    "- OVS Stats Dropped  \n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "ovs_stats_dropped('metrics_data')\n",
    "```\n",
    "\n",
    "- OVS Stats Errors\n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "ovs_stats_errors('metrics_data')\n",
    "```\n",
    "\n",
    "##### DPDK \n",
    "\n",
    "- DPDK Dropped  \n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "dpdk_dropped('metrics_data')\n",
    "```\n",
    "\n",
    "- DPDK Errors\n",
    "\n",
    "```py\n",
    "# Using CSV\n",
    "dpdk_errors('metrics_data')\n",
    "```\n",
    "\n",
    "\n",
    "\n",
    "##### RDT (Do not run yet)\n",
    "\n",
    "- Plot bytes\n",
    "\n",
    "```py\n",
    "#csv\n",
    "plot_rdt_bytes('metrics_data')\n",
    "```\n",
    "\n",
    "- Plot ipc values\n",
    "\n",
    "```py\n",
    "#csv\n",
    "plot_rdt_ipc('metrics_data')\n",
    "```\n",
    "\n",
    "- Memory bandwidth\n",
    "\n",
    "```py\n",
    "#csv\n",
    "get_rdt_memory_bandwidth('metrics_data')\n",
    "```\n",
    "\n",
    "##### Memory\n",
    "\n",
    "```py\n",
    "#csv\n",
    "get_memory_usage('metrics_data')\n",
    "```"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}