blob: b07bf0ed118bfe6d334b7f16a735631e10fa273f (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
#!/bin/sh
#
# rbd_mirror_stress.sh - stress test rbd-mirror daemon
#
# The following additional environment variables affect the test:
#
# RBD_MIRROR_REDUCE_WRITES - if not empty, don't run the stress bench write
# tool during the many image test
#
IMAGE_COUNT=50
export LOCKDEP=0
. $(dirname $0)/rbd_mirror_helpers.sh
create_snap()
{
local cluster=$1
local pool=$2
local image=$3
local snap_name=$4
rbd --cluster ${cluster} -p ${pool} snap create ${image}@${snap_name} \
--debug-rbd=20 --debug-journaler=20 2> ${TEMPDIR}/rbd-snap-create.log
}
compare_image_snaps()
{
local pool=$1
local image=$2
local snap_name=$3
local rmt_export=${TEMPDIR}/${CLUSTER2}-${pool}-${image}.export
local loc_export=${TEMPDIR}/${CLUSTER1}-${pool}-${image}.export
rm -f ${rmt_export} ${loc_export}
rbd --cluster ${CLUSTER2} -p ${pool} export ${image}@${snap_name} ${rmt_export}
rbd --cluster ${CLUSTER1} -p ${pool} export ${image}@${snap_name} ${loc_export}
cmp ${rmt_export} ${loc_export}
rm -f ${rmt_export} ${loc_export}
}
wait_for_pool_images()
{
local cluster=$1
local pool=$2
local image_count=$3
local s
local count
local last_count=0
while true; do
for s in `seq 1 40`; do
test $s -ne 1 && sleep 30
count=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'images: ' | cut -d' ' -f 2)
test "${count}" = "${image_count}" && return 0
# reset timeout if making forward progress
test $count -ne $last_count && break
done
test $count -eq $last_count && break
last_count=$count
done
rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
return 1
}
wait_for_pool_healthy()
{
local cluster=$1
local pool=$2
local s
local state
for s in `seq 1 40`; do
test $s -ne 1 && sleep 30
state=$(rbd --cluster ${cluster} -p ${pool} mirror pool status | grep 'health:' | cut -d' ' -f 2)
test "${state}" = "ERROR" && break
test "${state}" = "OK" && return 0
done
rbd --cluster ${cluster} -p ${pool} mirror pool status --verbose >&2
return 1
}
start_mirror ${CLUSTER1}
start_mirror ${CLUSTER2}
testlog "TEST: add image and test replay after client crashes"
image=test
create_image ${CLUSTER2} ${POOL} ${image} '512M'
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
for i in `seq 1 10`
do
stress_write_image ${CLUSTER2} ${POOL} ${image}
wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'master_position'
snap_name="snap${i}"
create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
compare_image_snaps ${POOL} ${image} ${snap_name}
done
for i in `seq 1 10`
do
snap_name="snap${i}"
remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
done
remove_image_retry ${CLUSTER2} ${POOL} ${image}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
testlog "TEST: create many images"
snap_name="snap"
for i in `seq 1 ${IMAGE_COUNT}`
do
image="image_${i}"
create_image ${CLUSTER2} ${POOL} ${image} '128M'
if [ -n "${RBD_MIRROR_REDUCE_WRITES}" ]; then
write_image ${CLUSTER2} ${POOL} ${image} 100
else
stress_write_image ${CLUSTER2} ${POOL} ${image}
fi
done
wait_for_pool_images ${CLUSTER2} ${POOL} ${IMAGE_COUNT}
wait_for_pool_healthy ${CLUSTER2} ${POOL}
wait_for_pool_images ${CLUSTER1} ${POOL} ${IMAGE_COUNT}
wait_for_pool_healthy ${CLUSTER1} ${POOL}
testlog "TEST: compare many images"
for i in `seq 1 ${IMAGE_COUNT}`
do
image="image_${i}"
create_snap ${CLUSTER2} ${POOL} ${image} ${snap_name}
wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
wait_for_snap_present ${CLUSTER1} ${POOL} ${image} ${snap_name}
compare_image_snaps ${POOL} ${image} ${snap_name}
done
testlog "TEST: delete many images"
for i in `seq 1 ${IMAGE_COUNT}`
do
image="image_${i}"
remove_snapshot ${CLUSTER2} ${POOL} ${image} ${snap_name}
remove_image_retry ${CLUSTER2} ${POOL} ${image}
done
testlog "TEST: image deletions should propagate"
wait_for_pool_images ${CLUSTER1} ${POOL} 0
wait_for_pool_healthy ${CLUSTER1} ${POOL} 0
for i in `seq 1 ${IMAGE_COUNT}`
do
image="image_${i}"
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
done
testlog "TEST: delete images during bootstrap"
set_pool_mirror_mode ${CLUSTER1} ${POOL} 'image'
set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image'
start_mirror ${CLUSTER1}
image=test
for i in `seq 1 10`
do
image="image_${i}"
create_image ${CLUSTER2} ${POOL} ${image} '512M'
enable_mirror ${CLUSTER2} ${POOL} ${image}
stress_write_image ${CLUSTER2} ${POOL} ${image}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present'
disable_mirror ${CLUSTER2} ${POOL} ${image}
wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
purge_snapshots ${CLUSTER2} ${POOL} ${image}
remove_image_retry ${CLUSTER2} ${POOL} ${image}
done
echo OK
|
