1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
|
#!/usr/bin/env python
# Copyright (c) Authors of Clover
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
import getopt
import subprocess
import sys
import time
import uuid
import yaml
#sys.path.insert(0, '..')
from clover.orchestration.kube_client import KubeClient
import clover.servicemesh.route_rules as rr
from clover.tools.validate_rr import ValidateWRR
from clover.tracing.tracing import Tracing
from validate_success import validate_perf
def _format_perf_data(perf_dict, dep_name, svc):
in_pod= None
out_pod = None
out_pod_list = []
for key, perf in perf_dict.items():
if key == 'in':
continue
elif key == 'out':
if 'out_svc' in perf:
out_pod = perf.get('out_svc')
elif 'out_svc' in perf:
if perf.get('out_svc') == svc:
in_pod = key
if out_pod:
out_pod_list = [key for key in perf_dict.keys() if out_pod in key.lower()]
if out_pod_list:
out_pod = out_pod_list[0]
print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name] + out_pod_list))
print("{: >20} {: >20} {: >20}".format(*[perf_dict[in_pod].get('average'),
perf_dict['in'].get('average'),
perf_dict[out_pod].get('average')]))
return
print("{: >20} {: >20} {: >20}".format(*[in_pod, dep_name, out_pod]))
print("{: >20} {: >20}".format(*[perf_dict[in_pod].get('average'),
perf_dict['in'].get('average')]))
def main(argv):
test_yaml = None
namespace = 'default'
tracing_port = 0
help_str = 'python fraser_a_b_test.py -t <test-yaml> -n <namespace> -p <tracing port>'
try:
opts, args = getopt.getopt(argv,"ht:n:p:",["test-yaml", "namespace", "tracing-port"])
except getopt.GetoptError:
print help_str
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print help_str
sys.exit()
elif opt in ("-t", "--test-yaml"):
test_yaml = str(arg)
elif opt in ("-n", "--namespace"):
namespace = str(arg)
elif opt in ("-p", "--tracing-port"):
tracing_port = int(arg)
if not test_yaml or tracing_port == 0:
print help_str
sys.exit(3)
with open(test_yaml) as fp:
test_params = yaml.load(fp)
'''
Steps:
(1) get version one info
(2) get version two info
(3) start version two
(4) validate version two pod and sidecar all up
(5) load A-B testing route rules
(6) execute traffic test script
(7) validate route rules traffic distribution
(8) validate version two success criteria
(9) if (8) works, change to version 2 only
(10) execute traffic test script
(11) validate route rules traffic distribution
'''
APP_BASE = 'test/app/'
POLICY_BASE = 'test/istio/'
SCRIPT_BASE = 'test/script/'
print('Current pods running at namespace %s' % namespace)
# as this is just for display purpose, we directly use kubectl get pods
cmd = 'kubectl get pods -n %s' % namespace
output = subprocess.check_output(cmd, shell=True)
print(output)
print('Current services running at namespace %s' % namespace)
cmd = 'kubectl get svc -n %s' % namespace
output = subprocess.check_output(cmd, shell=True)
print(output)
# service under test
test_svc = test_params.get('test-svc')
print('Service under test: %s' % test_svc)
k8s_client = KubeClient()
on, _ = k8s_client.check_pod_up('istio-sidecar-injector', 'istio-system')
print('Istio automatic sidecar injection is %s' % on)
dep_a_name = test_params.get('deployment-A')
dep_b = test_params.get('deployment-B')
dep_b_name = dep_b.get('name')
dep_b_yaml = APP_BASE + dep_b.get('manifest')
additional_deps = test_params.get('additional-deployments')
# TODO(s3wong): use istio-inject, then use kube_client to invoke
dep_list = []
print('Deploying %s...' % dep_b_name)
if not on:
cmd_temp = 'istioctl kube-inject -f %s > app/__tmp.yaml; kubectl apply -f app/__tmp.yaml; rm -f app/__tmp.yaml'
else:
cmd_temp = 'kubectl apply -f %s'
up, _ = k8s_client.check_pod_up(dep_b_name, namespace=namespace)
if up:
print('%s already has pod up, no need to spawn...' % dep_b_name)
else:
cmd = cmd_temp % dep_b_yaml
output = subprocess.check_output(cmd, shell=True)
print(output)
dep_list.append({'name': dep_b_name, 'up': False})
if additional_deps:
for dep in additional_deps:
dep_name = dep.get('name')
dep_yaml = APP_BASE + dep.get('manifest')
up, _ = k8s_client.check_pod_up(dep_name, namespace=namespace)
if up:
print('%s already has pod up, no need to spawn...' % dep_name)
else:
cmd = cmd_temp % dep_yaml
output = subprocess.check_output(cmd, shell=True)
print(output)
dep_list.append({'name': dep_name, 'up': False})
time.sleep(3)
wait_count = 0
continue_waiting = False
while wait_count < 5:
continue_waiting = False
for dep in dep_list:
if not dep.get('up'):
dep['up'], _ = k8s_client.check_pod_up(dep.get('name'), namespace=namespace)
if not dep['up']:
continue_waiting = True
if continue_waiting:
wait_count += 1
time.sleep(3)
else:
break
if continue_waiting:
print('Some pods are still not up after 15 seconds: %s' % dep_list)
sys.exit(4)
print('All pods are up')
cmd = 'kubectl get pods -n %s' % namespace
output = subprocess.check_output(cmd, shell=True)
print(output)
time.sleep(3)
a_b_test_rr_yaml = POLICY_BASE + test_params.get('ab-test-rr')
print('Loading route rules in %s' % a_b_test_rr_yaml)
ret = rr.load_route_rules(a_b_test_rr_yaml)
print('Route rules are now %s' % rr.get_route_rules())
time.sleep(5)
redis_pod = k8s_client.find_pod_by_name('redis')
if not redis_pod:
print('redis not running in default namespace')
sys.exit(6)
redis_ip = redis_pod.get('pod_ip')
tracing = Tracing(tracing_ip='localhost',
tracing_port=str(tracing_port),
redis_ip=redis_ip)
# turn off tracing to redis for warm up run
tracing.use_redis = False
traffic_test_dict = test_params.get('traffic-test')
traffic_test_script = traffic_test_dict.get('name')
traffic_test_params = traffic_test_dict.get('params')
cmd = SCRIPT_BASE + traffic_test_script
if traffic_test_params:
for param in traffic_test_params:
cmd = cmd + ' ' + str(param)
print('Execute traffic test %s' % cmd)
'''
print('Warming up for route rules to take place')
try:
output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError, e:
print('%s returns error %s' % e.output)
print(output)
print('Running recorded traffic test...')
'''
time.sleep(30)
tracing.use_redis = True
test_id = uuid.uuid4()
rr.set_route_rules(test_id)
tracing.setTest(test_id)
# wait 1 sec to avoid missing the first test result
time.sleep(1)
try:
output = subprocess.check_output(cmd, shell=True)
except subprocess.CalledProcessError, e:
print('non zero return value on traffic script: %s, ignoring...' % e.output)
print(output)
time.sleep(30)
traces = tracing.getTraces(test_svc, 0)
tracing.outTraces(traces)
time.sleep(3)
print('Validating route rules...')
validate_wrr = ValidateWRR(test_id, redis_ip=redis_ip)
ret, errors = validate_wrr.validate(test_svc)
# TODO(s3wong): for now, route rules failure seems more like a warning
if ret:
print('Route rules for service %s validated' % test_svc)
else:
print('Route rules for service %s validation failed' % test_svc)
for err in errors:
print err
success_factors = test_params.get('success')
if success_factors:
criteria = success_factors.get('criteria')
success_check = True
for criterion in criteria:
c_type = criterion.get('type')
if c_type == 'performance':
condition = int(criterion.get('condition'))
ret_dict = validate_perf(tracing, test_id, test_svc,
dep_a_name, dep_b_name)
# print performance data
_format_perf_data(ret_dict.get(dep_a_name), dep_a_name, test_svc)
print('\n')
_format_perf_data(ret_dict.get(dep_b_name), dep_b_name, test_svc)
ret = (ret_dict.get(dep_b_name).get('in').get('average') <= \
(ret_dict.get(dep_a_name).get('in').get('average') * condition / 100))
if not ret:
print('Performance check failed')
success_check = False
break
else:
print('Performance check succeed')
'''
elif c_type == 'services':
srv_list = criterion.get('services')
ret = check_services_traverse(tracing, test_id, test_svc,
dep_b_name, srv_list)
if not ret:
print('Additional services traversal test failed')
success_check = False
break
else:
print('Additional services traversal test succeed')
'''
if success_check:
actions = success_factors.get('action')
else:
failed = success_factors.get('failed')
actions = failed.get('action')
for action in actions:
action_type = action.get('type')
if action_type == 'commit' or action_type == 'rollback':
rr.delete_route_rules(a_b_test_rr_yaml, namespace)
ret = rr.load_route_rules(POLICY_BASE + action.get('routerule'))
if ret:
print('loading route rule %s succeed' % action.get('routerule'))
if __name__ == "__main__":
main(sys.argv[1:])
|