1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileStatus;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.testclassification.SmallTests;
34 import org.apache.hadoop.hbase.util.FSUtils;
35 import org.apache.hadoop.hbase.util.StoppableImplementation;
36 import org.junit.After;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39 import org.mockito.Mockito;
40 import org.mockito.invocation.InvocationOnMock;
41 import org.mockito.stubbing.Answer;
42
43 @Category(SmallTests.class)
44 public class TestCleanerChore {
45
46 private static final Log LOG = LogFactory.getLog(TestCleanerChore.class);
47 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
48
49 @After
50 public void cleanup() throws Exception {
51
52 UTIL.cleanupTestDir();
53 }
54
55
56 @Test
57 public void testSavesFilesOnRequest() throws Exception {
58 Stoppable stop = new StoppableImplementation();
59 Configuration conf = UTIL.getConfiguration();
60 Path testDir = UTIL.getDataTestDir();
61 FileSystem fs = UTIL.getTestFileSystem();
62 String confKey = "hbase.test.cleaner.delegates";
63 conf.set(confKey, NeverDelete.class.getName());
64
65 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
66
67
68 Path parent = new Path(testDir, "parent");
69 Path file = new Path(parent, "someFile");
70 fs.mkdirs(parent);
71
72 fs.create(file).close();
73 assertTrue("Test file didn't get created.", fs.exists(file));
74
75
76 chore.chore();
77
78
79 assertTrue("File didn't get deleted", fs.exists(file));
80 assertTrue("Empty directory didn't get deleted", fs.exists(parent));
81 }
82
83 @Test
84 public void testDeletesEmptyDirectories() throws Exception {
85 Stoppable stop = new StoppableImplementation();
86 Configuration conf = UTIL.getConfiguration();
87 Path testDir = UTIL.getDataTestDir();
88 FileSystem fs = UTIL.getTestFileSystem();
89 String confKey = "hbase.test.cleaner.delegates";
90 conf.set(confKey, AlwaysDelete.class.getName());
91
92 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
93
94
95 Path parent = new Path(testDir, "parent");
96 Path child = new Path(parent, "child");
97 Path emptyChild = new Path(parent, "emptyChild");
98 Path file = new Path(child, "someFile");
99 fs.mkdirs(child);
100 fs.mkdirs(emptyChild);
101
102 fs.create(file).close();
103
104 Path topFile = new Path(testDir, "topFile");
105 fs.create(topFile).close();
106 assertTrue("Test file didn't get created.", fs.exists(file));
107 assertTrue("Test file didn't get created.", fs.exists(topFile));
108
109
110 chore.chore();
111
112
113 assertFalse("File didn't get deleted", fs.exists(topFile));
114 assertFalse("File didn't get deleted", fs.exists(file));
115 assertFalse("Empty directory didn't get deleted", fs.exists(child));
116 assertFalse("Empty directory didn't get deleted", fs.exists(parent));
117 }
118
119
120
121
122
123
124 @Test
125 public void testDoesNotCheckDirectories() throws Exception {
126 Stoppable stop = new StoppableImplementation();
127 Configuration conf = UTIL.getConfiguration();
128 Path testDir = UTIL.getDataTestDir();
129 FileSystem fs = UTIL.getTestFileSystem();
130 String confKey = "hbase.test.cleaner.delegates";
131 conf.set(confKey, AlwaysDelete.class.getName());
132
133 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
134
135 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
136 AlwaysDelete spy = Mockito.spy(delegate);
137 chore.cleanersChain.set(0, spy);
138
139
140 Path parent = new Path(testDir, "parent");
141 Path file = new Path(parent, "someFile");
142 fs.mkdirs(parent);
143 assertTrue("Test parent didn't get created.", fs.exists(parent));
144
145 fs.create(file).close();
146 assertTrue("Test file didn't get created.", fs.exists(file));
147
148 FileStatus fStat = fs.getFileStatus(parent);
149 chore.chore();
150
151 Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
152 Mockito.reset(spy);
153 }
154
155 @Test
156 public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
157 Stoppable stop = new StoppableImplementation();
158 Configuration conf = UTIL.getConfiguration();
159 Path testDir = UTIL.getDataTestDir();
160 FileSystem fs = UTIL.getTestFileSystem();
161 String confKey = "hbase.test.cleaner.delegates";
162 conf.set(confKey, AlwaysDelete.class.getName());
163
164 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
165
166
167 Path topFile = new Path(testDir, "topFile");
168 fs.create(topFile).close();
169 assertTrue("Test file didn't get created.", fs.exists(topFile));
170
171
172 stop.stop("testing stop");
173
174
175 chore.chore();
176
177
178 assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
179 }
180
181
182
183
184
185
186 @Test
187 public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
188 Stoppable stop = new StoppableImplementation();
189 Configuration conf = UTIL.getConfiguration();
190 final Path testDir = UTIL.getDataTestDir();
191 final FileSystem fs = UTIL.getTestFileSystem();
192 String confKey = "hbase.test.cleaner.delegates";
193 conf.set(confKey, AlwaysDelete.class.getName());
194
195 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
196
197 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
198 AlwaysDelete spy = Mockito.spy(delegate);
199 chore.cleanersChain.set(0, spy);
200
201
202 final Path parent = new Path(testDir, "parent");
203 Path file = new Path(parent, "someFile");
204 fs.mkdirs(parent);
205
206 fs.create(file).close();
207 assertTrue("Test file didn't get created.", fs.exists(file));
208 final Path addedFile = new Path(parent, "addedFile");
209
210
211 Mockito.doAnswer(new Answer<Boolean>() {
212 @Override
213 public Boolean answer(InvocationOnMock invocation) throws Throwable {
214 fs.create(addedFile).close();
215 FSUtils.logFileSystemState(fs, testDir, LOG);
216 return (Boolean) invocation.callRealMethod();
217 }
218 }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
219
220
221 chore.chore();
222
223
224 assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
225 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
226 assertFalse("Original file unexpectedly retained", fs.exists(file));
227 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
228 Mockito.reset(spy);
229 }
230
231
232
233
234
235
236
237
238
239
240 @Test
241 public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
242 Stoppable stop = new StoppableImplementation();
243
244
245 HBaseTestingUtility localUtil = new HBaseTestingUtility();
246 Configuration conf = localUtil.getConfiguration();
247 final Path testDir = UTIL.getDataTestDir();
248 final FileSystem fs = UTIL.getTestFileSystem();
249 LOG.debug("Writing test data to: " + testDir);
250 String confKey = "hbase.test.cleaner.delegates";
251 conf.set(confKey, AlwaysDelete.class.getName());
252
253 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
254
255 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
256 AlwaysDelete spy = Mockito.spy(delegate);
257 chore.cleanersChain.set(0, spy);
258
259
260 final Path parent = new Path(testDir, "parent");
261 Path file = new Path(parent, "someFile");
262 fs.mkdirs(parent);
263
264 fs.create(file).close();
265 assertTrue("Test file didn't get created.", fs.exists(file));
266 final Path racyFile = new Path(parent, "addedFile");
267
268
269 Mockito.doAnswer(new Answer<Boolean>() {
270 @Override
271 public Boolean answer(InvocationOnMock invocation) throws Throwable {
272 fs.create(racyFile).close();
273 FSUtils.logFileSystemState(fs, testDir, LOG);
274 return (Boolean) invocation.callRealMethod();
275 }
276 }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
277
278
279 if (chore.checkAndDeleteDirectory(parent)) {
280 throw new Exception(
281 "Reported success deleting directory, should have failed when adding file mid-iteration");
282 }
283
284
285 assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
286 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
287 assertFalse("Original file unexpectedly retained", fs.exists(file));
288 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
289 }
290
291 private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
292
293 public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
294 Path oldFileDir, String confkey) {
295 super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey);
296 }
297
298
299 @Override
300 protected boolean validate(Path file) {
301 return true;
302 }
303 };
304
305 public static class AlwaysDelete extends BaseHFileCleanerDelegate {
306 @Override
307 public boolean isFileDeletable(FileStatus fStat) {
308 return true;
309 }
310 }
311
312 public static class NeverDelete extends BaseHFileCleanerDelegate {
313 @Override
314 public boolean isFileDeletable(FileStatus fStat) {
315 return false;
316 }
317 }
318 }