1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
|
"""
Exercise the MDS's auto repair functions
"""
import logging
import time
from teuthology.orchestra.run import CommandFailedError
from tasks.cephfs.cephfs_test_case import CephFSTestCase
log = logging.getLogger(__name__)
# Arbitrary timeouts for operations involving restarting
# an MDS or waiting for it to come up
MDS_RESTART_GRACE = 60
class TestMDSAutoRepair(CephFSTestCase):
def test_backtrace_repair(self):
"""
MDS should verify/fix backtrace on fetch dirfrag
"""
self.mount_a.run_shell(["mkdir", "testdir1"])
self.mount_a.run_shell(["touch", "testdir1/testfile"])
dir_objname = "{:x}.00000000".format(self.mount_a.path_to_ino("testdir1"))
# drop inodes caps
self.mount_a.umount_wait()
# flush journal entries to dirfrag objects, and expire journal
self.fs.mds_asok(['flush', 'journal'])
# Restart the MDS to drop the metadata cache (because we expired the journal,
# nothing gets replayed into cache on restart)
self.fs.mds_stop()
self.fs.mds_fail_restart()
self.fs.wait_for_daemons()
# remove testdir1's backtrace
self.fs.rados(["rmxattr", dir_objname, "parent"])
# readdir (fetch dirfrag) should fix testdir1's backtrace
self.mount_a.mount()
self.mount_a.wait_until_mounted()
self.mount_a.run_shell(["ls", "testdir1"])
# flush journal entries to dirfrag objects
self.fs.mds_asok(['flush', 'journal'])
# check if backtrace exists
self.fs.rados(["getxattr", dir_objname, "parent"])
def test_mds_readonly(self):
"""
test if MDS behave correct when it's readonly
"""
# operation should successd when MDS is not readonly
self.mount_a.run_shell(["touch", "test_file1"])
writer = self.mount_a.write_background(loop=True)
time.sleep(10)
self.assertFalse(writer.finished)
# force MDS to read-only mode
self.fs.mds_asok(['force_readonly'])
time.sleep(10)
# touching test file should fail
try:
self.mount_a.run_shell(["touch", "test_file1"])
except CommandFailedError:
pass
else:
self.assertTrue(False)
# background writer also should fail
self.assertTrue(writer.finished)
# The MDS should report its readonly health state to the mon
self.wait_for_health("MDS_READ_ONLY", timeout=30)
# restart mds to make it writable
self.fs.mds_fail_restart()
self.fs.wait_for_daemons()
self.wait_for_health_clear(timeout=30)
|
