#!/usr/bin/env python
#
# jose.lausuch@ericsson.com
# valentin.boucher@orange.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
#
from datetime import datetime as dt
import json
import os
import os.path
import re
import requests
import shutil
import socket
import subprocess
import sys
import urllib2
import yaml
from git import Repo
import functest.ci.tier_builder as tb
""" global variables """
REPOS_DIR = os.getenv('repos_dir')
FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
# ----------------------------------------------------------
#
# INTERNET UTILS
#
# -----------------------------------------------------------
def check_internet_connectivity(url='http://www.opnfv.org/'):
"""
Check if there is access to the internet
"""
try:
urllib2.urlopen(url, timeout=5)
return True
except urllib2.URLError:
return False
def download_url(url, dest_path):
"""
Download a file to a destination path given a URL
"""
name = url.rsplit('/')[-1]
dest = dest_path + "/" + name
try:
response = urllib2.urlopen(url)
except (urllib2.HTTPError, urllib2.URLError):
return False
with open(dest, 'wb') as f:
shutil.copyfileobj(response, f)
return True
# ----------------------------------------------------------
#
# CI UTILS
#
# -----------------------------------------------------------
def get_git_branch(repo_path):
"""
Get git branch name
"""
repo = Repo(repo_path)
branch = repo.active_branch
return branch.name
def get_installer_type(logger=None):
"""
Get installer type (fuel, apex, joid, compass)
"""
try:
installer = os.environ['INSTALLER_TYPE']
except KeyError:
if logger:
logger.error("Impossible to retrieve the installer type")
installer = "Unknown_installer"
return installer
def get_scenario(logger=None):
"""
Get scenario
"""
try:
scenario = os.environ['DEPLOY_SCENARIO']
except KeyError:
if logger:
logger.error("Impossible to retrieve the scenario")
scenario = "Unknown_scenario"
return scenario
def get_version(logger=None):
"""
Get version
"""
# Use the build tag to retrieve the version
# By default version is unknown
# if launched through CI the build tag has the following format
# jenkins-<project>-<installer>-<pod>-<job>-<branch>-<id>
# e.g. jenkins-functest-fuel-opnfv-jump-2-daily-master-190
# use regex to match branch info
rule = "daily-(.+?)-[0-9]*"
build_tag = get_build_tag(logger)
m = re.search(rule, build_tag)
if m:
return m.group(1)
else:
return "unknown"
def get_pod_name(logger=None):
"""
Get PoD Name from env variable NODE_NAME
"""
try:
return os.environ['NODE_NAME']
except KeyError:
if logger:
logger.error(
"Unable to retrieve the POD name from environment. " +
"Using pod name 'unknown-pod'")
return "unknown-pod"
def get_build_tag(logger=None):
"""
Get build tag of jenkins jobs
"""
try:
build_tag = os.environ['BUILD_TAG']
except KeyError:
if logger:
logger.error("Impossible to retrieve the build tag")
build_tag = "unknown_build_tag"
return build_tag
def get_db_url(logger=None):
"""
Returns DB URL
"""
with open(os.environ["CONFIG_FUNCTEST_YAML"]) as f:
functest_yaml = yaml.safe_load(f)
f.close()
db_url = functest_yaml.get("results").get("test_db_url")
return db_url
def push_results_to_db(project, case_name, logger,
start_date, stop_date, criteria, details):
"""
POST results to the Result target DB
"""
# Retrieve params from CI and conf
url = get_db_url(logger) + "/results"
installer = get_installer_type(logger)
scenario = get_scenario(logger)
version = get_version(logger)
pod_name = get_pod_name(logger)
build_tag = get_build_tag(logger)
test_start = dt.fromtimestamp(start_date).strftime('%Y-%m-%d %H:%M:%S')
test_stop = dt.fromtimestamp(stop_date).strftime('%Y-%m-%d %H:%M:%S')
params = {"project_name": project, "case_name": case_name,
"pod_name": pod_name, "installer": installer,
"version": version, "scenario": scenario, "criteria": criteria,
"build_tag": build_tag, "start_date": test_start,
"stop_date": test_stop, "details": details}
headers = {'Content-Type': 'application/json'}
try:
r = requests.post(url, data=json.dumps(params), headers=headers)
if logger:
logger.debug(r)
return True
except Exception, e:
print ("Error [push_results_to_db('%s', '%s', '%s', " +
"'%s', '%s', '%s', '%s', '%s', '%s')]:" %
(url, project, case_name, pod_name, version,
scenario, criteria, build_tag, details)), e
return False
def get_resolvconf_ns():
"""
Get nameservers from current resolv.conf
"""
nameservers = []
rconf = open("/etc/resolv.conf", "r")
line = rconf.readline()
while line:
ip = re.search(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b", line)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if ip:
result = sock.connect_ex((ip.group(), 53))
if result == 0:
nameservers.append(ip.group())
line = rconf.readline()
return nameservers
def get_ci_envvars():
"""
Get the CI env variables
"""
ci_env_var = {
{
"__inputs": [
{
"name": "DS_YARDSTICK",
"label": "yardstick",
"description": "",
"type": "datasource",
"pluginId": "influxdb",
"pluginName": "InfluxDB"
}
],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "4.4.3"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
},
{
"type": "datasource",
"id": "influxdb",
"name": "InfluxDB",
"version": "1.0.0"
},
{
"type": "panel",
"id": "text",
"name": "Text",
"version": ""
}
],
"annotations": {
"list": []
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"hideControls": false,
"id": null,
"links": [],
"refresh": false,
"rows": [
{
"collapse": false,
"height": "100px",
"panels": [
{
"content": "<h5 style=\"font-family:Verdana\"> <a style=\"color:#31A7D3\"><a style=\"font: 32px '#31A7D3'\"><center>OPNFV_Yardstick_NSB_PROX_BM_BNG-4Port_Test</center> </a></h5>\n<center>\n<p>The BNG workload converts packets from QinQ to GRE tunnels, handles routing, adds/removes MPLS tags. </p>\n</center>",
"editable": true,
"error": false,
"id": 3,
"links": [],
"mode": "html",
"span": 12,
"title": "",
"type": "text"
},
{
"content": "<h5 style=\"font-family:Verdana\"> <a style=\"color:#31A7D3\"><a style=\"font: 22px '#31A7D3'\"><center>Prox BNG VNF stats</center> </a></h5>\n",
"editable": true,
"error": false,
"height": "40",
"id": 7,
"links": [],
"mode": "html",
"span": 12,
"title": "",
"type": "text"
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "Row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "300px",
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_YARDSTICK}",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"height": "300",
"id": 4,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"rightSide": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 1,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "VNF packets Forward",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"vnf__0.packets_fwd"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "VNF packets in",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"policy": "default",
"refId": "C",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"vnf__0.packets_in"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "VNF packets dropped",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"vnf__0.packets_dropped"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": [
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
"value": 2
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
"value": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "BNG VNF stats",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Packets",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"content": "<h5 style=\"font-family:Verdana\"> <a style=\"color:#31A7D3\"><a style=\"font: 22px '#31A7D3'\"><center>Prox BNG Traffic Gen stats</center> </a></h5>\n",
"editable": true,
"error": false,
"height": "40",
"id": 8,
"links": [],
"mode": "html",
"span": 12,
"title": "",
"type": "text"
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "300px",
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_YARDSTICK}",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 6,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "TG xe-0 Out packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe0.out_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-1 Out packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe1.out_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-2 Out packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "C",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe2.out_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-3 Out packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "D",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe3.out_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": [
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
"value": 2
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
"value": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "TG Port Stats: Out packets",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Packets",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_YARDSTICK}",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 9,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "TG xe-0 in packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe0.in_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-1 in packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe1.in_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-2 in packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "C",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe2.in_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG xe-3 in packets",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "D",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.xe3.in_packets"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": [
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
"value": 2
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
"value": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "TG Port Stats: In packets",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Packets",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_YARDSTICK}",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "TG Theoritical TX Throughput",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.TxThroughput"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
},
{
"alias": "TG RX Throughput",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "B",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.RxThroughput"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": [
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
"value": 2
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
"value": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "TG Tx vs Rx Throughput",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "Throughput in MPPS",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_YARDSTICK}",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"id": 5,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "Packet Size",
"dsType": "influxdb",
"groupBy": [
{
"params": [
"$interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"measurement": "tc_prox_baremetal_bng-4",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"tg__0.PktSize"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": [
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(216, 200, 27, 0.27)",
"op": "gt",
"value": 2
},
{
"colorMode": "custom",
"fill": true,
"fillColor": "rgba(234, 112, 112, 0.22)",
"op": "gt",
"value": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "Packet size",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": "Packet Size",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
},
{
"collapse": false,
"height": "250px",
"panels": [],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "New row",
"titleSize": "h6"
}
],
"schemaVersion": 14,
"style": "dark",
"tags": [
"yardstick",
"NSB",
"Prox",
"4Port",
"BM",
"BNG"
],
"templating": {
"list": []
},
"time": {
"from": "2017-08-11T08:30:38.655Z",
"to": "2017-08-11T08:33:45.419Z"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "browser",
"title": "Prox_BM_BNG-4Port",
"version": 5
}
