1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
from urllib2 import Request, urlopen, URLError
import json
import jinja2
import reportingConf as conf
import reportingUtils as utils
installers = conf.installers
items = ["tests", "Success rate", "duration"]
PERIOD = conf.PERIOD
criteria_nb_test = 165
criteria_duration = 1800
criteria_success_rate = 90
logger = utils.getLogger("Tempest")
logger.info("************************************************")
logger.info("* Generating reporting Tempest_smoke_serial *")
logger.info("* Data retention = %s days *" % PERIOD)
logger.info("* *")
logger.info("************************************************")
logger.info("Success criteria:")
logger.info("nb tests executed > %s s " % criteria_nb_test)
logger.info("test duration < %s s " % criteria_duration)
logger.info("success rate > %s " % criteria_success_rate)
for installer in installers:
# we consider the Tempest results of the last PERIOD days
url = conf.URL_BASE + "?case=tempest_smoke_serial"
request = Request(url + '&period=' + str(PERIOD) +
'&installer=' + installer + '&version=master')
logger.info("Search tempest_smoke_serial results for installer %s"
% installer)
try:
response = urlopen(request)
k = response.read()
results = json.loads(k)
except URLError, e:
logger.error("Error code: %s" % e)
test_results = results['results']
scenario_results = {}
criteria = {}
errors = {}
for r in test_results:
# Retrieve all the scenarios per installer
# In Brahmaputra use version
# Since Colorado use scenario
if not r['scenario'] in scenario_results.keys():
scenario_results[r['scenario']] = []
scenario_results[r['scenario']].append(r)
for s, s_result in scenario_results.items():
scenario_results[s] = s_result[0:5]
# For each scenario, we build a result object to deal with
# results, criteria and error handling
for result in scenario_results[s]:
result["start_date"] = result["start_date"].split(".")[0]
# retrieve results
# ****************
nb_tests_run = result['details']['tests']
nb_tests_failed = result['details']['failures']
if nb_tests_run != 0:
success_rate = 100*(int(nb_tests_run) -
int(nb_tests_failed)) / int(nb_tests_run)
else:
success_rate = 0
result['details']["tests"] = nb_tests_run
result['details']["Success rate"] = str(success_rate) + "%"
# Criteria management
# *******************
crit_tests = False
crit_rate = False
crit_time = False
# Expect that at least 165 tests are run
if nb_tests_run >= criteria_nb_test:
crit_tests = True
# Expect that at least 90% of success
if success_rate >= criteria_success_rate:
crit_rate = True
# Expect that the suite duration is inferior to 30m
if result['details']['duration'] < criteria_duration:
crit_time = True
result['criteria'] = {'tests': crit_tests,
'Success rate': crit_rate,
'duration': crit_time}
try:
logger.debug("Scenario %s, Installer %s"
% (s_result[1]['scenario'], installer))
logger.debug("Nb Test run: %s" % nb_tests_run)
logger.debug("Test duration: %s"
% result['details']['duration'])
logger.debug("Success rate: %s" % success_rate)
except:
logger.error("Data format error")
# Error management
# ****************
try:
errors = result['details']['errors']
result['errors'] = errors.replace('{0}', '')
except:
logger.error("Error field not present (Brahamputra runs?)")
templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH)
templateEnv = jinja2.Environment(loader=templateLoader)
TEMPLATE_FILE = "/template/index-tempest-tmpl.html"
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(scenario_results=scenario_results,
items=items,
installer=installer)
with open(conf.REPORTING_PATH + "/release/master/index-tempest-" +
installer + ".html", "wb") as fh:
fh.write(outputText)
logger.info("Tempest automatic reporting succesfully generated.")
|
