1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import static org.apache.hadoop.hbase.util.HFileArchiveTestingUtil.assertArchiveEqualToOriginal;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25 import static org.mockito.Mockito.doReturn;
26 import static org.mockito.Mockito.spy;
27
28 import java.io.IOException;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.SortedMap;
32 import java.util.TreeMap;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.NamespaceDescriptor;
48 import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
49 import org.apache.hadoop.hbase.Server;
50 import org.apache.hadoop.hbase.ServerName;
51 import org.apache.hadoop.hbase.TableDescriptors;
52 import org.apache.hadoop.hbase.catalog.CatalogTracker;
53 import org.apache.hadoop.hbase.catalog.MetaMockingUtil;
54 import org.apache.hadoop.hbase.client.HConnection;
55 import org.apache.hadoop.hbase.client.HConnectionManager;
56 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.executor.ExecutorService;
59 import org.apache.hadoop.hbase.io.Reference;
60 import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
61 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
62 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
63 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
64 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
65 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateResponse;
66 import org.apache.hadoop.hbase.regionserver.HStore;
67 import org.apache.hadoop.hbase.testclassification.SmallTests;
68 import org.apache.hadoop.hbase.util.Bytes;
69 import org.apache.hadoop.hbase.util.FSUtils;
70 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
71 import org.apache.hadoop.hbase.util.Triple;
72 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
73 import org.junit.Test;
74 import org.junit.experimental.categories.Category;
75 import org.mockito.Mockito;
76
77 import com.google.protobuf.RpcController;
78 import com.google.protobuf.Service;
79 import com.google.protobuf.ServiceException;
80
81 @Category(SmallTests.class)
82 public class TestCatalogJanitor {
83 private static final Log LOG = LogFactory.getLog(TestCatalogJanitor.class);
84
85
86
87
88
89 class MockServer implements Server {
90 private final HConnection connection;
91 private final Configuration c;
92 private final CatalogTracker ct;
93
94 MockServer(final HBaseTestingUtility htu)
95 throws NotAllMetaRegionsOnlineException, IOException, InterruptedException {
96 this.c = htu.getConfiguration();
97 ClientProtos.ClientService.BlockingInterface ri =
98 Mockito.mock(ClientProtos.ClientService.BlockingInterface.class);
99 MutateResponse.Builder builder = MutateResponse.newBuilder();
100 builder.setProcessed(true);
101 try {
102 Mockito.when(ri.mutate(
103 (RpcController)Mockito.any(), (MutateRequest)Mockito.any())).
104 thenReturn(builder.build());
105 } catch (ServiceException se) {
106 throw ProtobufUtil.getRemoteException(se);
107 }
108
109
110
111 this.connection =
112 HConnectionTestingUtility.getMockedConnectionAndDecorate(this.c,
113 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class), ri,
114 ServerName.valueOf("example.org,12345,6789"),
115 HRegionInfo.FIRST_META_REGIONINFO);
116
117 FileSystem fs = FileSystem.get(this.c);
118 Path rootdir = FSUtils.getRootDir(this.c);
119 FSUtils.setRootDir(this.c, rootdir);
120 this.ct = Mockito.mock(CatalogTracker.class);
121 AdminProtos.AdminService.BlockingInterface hri =
122 Mockito.mock(AdminProtos.AdminService.BlockingInterface.class);
123 Mockito.when(this.ct.getConnection()).thenReturn(this.connection);
124 Mockito.when(ct.waitForMetaServerConnection(Mockito.anyLong())).thenReturn(hri);
125 }
126
127 @Override
128 public CatalogTracker getCatalogTracker() {
129 return this.ct;
130 }
131
132 @Override
133 public Configuration getConfiguration() {
134 return this.c;
135 }
136
137 @Override
138 public ServerName getServerName() {
139 return ServerName.valueOf("mockserver.example.org", 1234, -1L);
140 }
141
142 @Override
143 public ZooKeeperWatcher getZooKeeper() {
144 return null;
145 }
146
147 @Override
148 public void abort(String why, Throwable e) {
149
150 }
151
152 @Override
153 public boolean isAborted() {
154 return false;
155 }
156
157 @Override
158 public boolean isStopped() {
159 return false;
160 }
161
162 @Override
163 public void stop(String why) {
164 if (this.ct != null) {
165 this.ct.stop();
166 }
167 if (this.connection != null) {
168 HConnectionManager.deleteConnection(this.connection.getConfiguration());
169 }
170 }
171 }
172
173
174
175
176 class MockMasterServices implements MasterServices {
177 private final MasterFileSystem mfs;
178 private final AssignmentManager asm;
179
180 MockMasterServices(final Server server) throws IOException {
181 this.mfs = new MasterFileSystem(server, this, false);
182 this.asm = Mockito.mock(AssignmentManager.class);
183 }
184
185 @Override
186 public void checkTableModifiable(TableName tableName) throws IOException {
187
188 }
189
190 @Override
191 public void createTable(HTableDescriptor desc, byte[][] splitKeys)
192 throws IOException {
193
194 }
195
196 @Override
197 public AssignmentManager getAssignmentManager() {
198 return this.asm;
199 }
200
201 @Override
202 public ExecutorService getExecutorService() {
203 return null;
204 }
205
206 @Override
207 public MasterFileSystem getMasterFileSystem() {
208 return this.mfs;
209 }
210
211 @Override
212 public MasterCoprocessorHost getCoprocessorHost() {
213 return null;
214 }
215
216 @Override
217 public ServerManager getServerManager() {
218 return null;
219 }
220
221 @Override
222 public ZooKeeperWatcher getZooKeeper() {
223 return null;
224 }
225
226 @Override
227 public CatalogTracker getCatalogTracker() {
228 return null;
229 }
230
231 @Override
232 public Configuration getConfiguration() {
233 return mfs.conf;
234 }
235
236 @Override
237 public ServerName getServerName() {
238 return null;
239 }
240
241 @Override
242 public void abort(String why, Throwable e) {
243
244 }
245
246 @Override
247 public boolean isAborted() {
248 return false;
249 }
250
251 private boolean stopped = false;
252
253 @Override
254 public void stop(String why) {
255 stopped = true;
256 }
257
258 @Override
259 public boolean isStopped() {
260 return stopped;
261 }
262
263 @Override
264 public TableDescriptors getTableDescriptors() {
265 return new TableDescriptors() {
266 @Override
267 public HTableDescriptor remove(TableName tablename) throws IOException {
268
269 return null;
270 }
271
272 @Override
273 public Map<String, HTableDescriptor> getAll() throws IOException {
274
275 return null;
276 }
277
278 @Override
279 public HTableDescriptor get(TableName tablename)
280 throws IOException {
281 return createHTableDescriptor();
282 }
283
284 @Override
285 public Map<String, HTableDescriptor> getByNamespace(String name) throws IOException {
286 return null;
287 }
288
289 @Override
290 public void add(HTableDescriptor htd) throws IOException {
291
292
293 }
294
295 @Override
296 public void setCacheOn() throws IOException {
297 }
298
299 @Override
300 public void setCacheOff() throws IOException {
301 }
302 };
303 }
304
305 @Override
306 public boolean isServerShutdownHandlerEnabled() {
307 return true;
308 }
309
310 @Override
311 public boolean registerService(Service instance) {
312 return false;
313 }
314
315 @Override
316 public void createNamespace(NamespaceDescriptor descriptor) throws IOException {
317
318 }
319
320 @Override
321 public void modifyNamespace(NamespaceDescriptor descriptor) throws IOException {
322
323 }
324
325 @Override
326 public void deleteNamespace(String name) throws IOException {
327
328 }
329
330 @Override
331 public NamespaceDescriptor getNamespaceDescriptor(String name) throws IOException {
332 return null;
333 }
334
335 @Override
336 public List<NamespaceDescriptor> listNamespaceDescriptors() throws IOException {
337 return null;
338 }
339
340 @Override
341 public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
342 return null;
343 }
344
345 @Override
346 public List<TableName> listTableNamesByNamespace(String name) throws IOException {
347 return null;
348 }
349
350 @Override
351 public void deleteTable(TableName tableName) throws IOException { }
352
353 @Override
354 public void truncateTable(TableName tableName, boolean preserveSplits) throws IOException { }
355
356
357 @Override
358 public void modifyTable(TableName tableName, HTableDescriptor descriptor)
359 throws IOException { }
360
361 @Override
362 public void enableTable(TableName tableName) throws IOException { }
363
364 @Override
365 public void disableTable(TableName tableName) throws IOException { }
366
367 @Override
368 public void addColumn(TableName tableName, HColumnDescriptor column)
369 throws IOException { }
370
371 @Override
372 public void modifyColumn(TableName tableName, HColumnDescriptor descriptor)
373 throws IOException { }
374
375 @Override
376 public void deleteColumn(TableName tableName, byte[] columnName)
377 throws IOException { }
378
379 @Override
380 public TableLockManager getTableLockManager() {
381 return null;
382 }
383
384 @Override
385 public void dispatchMergingRegions(HRegionInfo region_a, HRegionInfo region_b,
386 boolean forcible) throws IOException {
387 }
388
389 @Override
390 public boolean isInitialized() {
391
392 return false;
393 }
394 }
395
396 @Test
397 public void testCleanParent() throws IOException, InterruptedException {
398 HBaseTestingUtility htu = new HBaseTestingUtility();
399 setRootDirAndCleanIt(htu, "testCleanParent");
400 Server server = new MockServer(htu);
401 try {
402 MasterServices services = new MockMasterServices(server);
403 CatalogJanitor janitor = new CatalogJanitor(server, services);
404
405 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
406 htd.addFamily(new HColumnDescriptor("f"));
407 HRegionInfo parent =
408 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
409 Bytes.toBytes("eee"));
410 HRegionInfo splita =
411 new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
412 Bytes.toBytes("ccc"));
413 HRegionInfo splitb =
414 new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
415 Bytes.toBytes("eee"));
416
417
418 Result r = createResult(parent, splita, splitb);
419
420 Path rootdir = services.getMasterFileSystem().getRootDir();
421 Path tabledir =
422 FSUtils.getTableDir(rootdir, htd.getTableName());
423 Path storedir = HStore.getStoreHomedir(tabledir, splita,
424 htd.getColumnFamilies()[0].getName());
425 Reference ref = Reference.createTopReference(Bytes.toBytes("ccc"));
426 long now = System.currentTimeMillis();
427
428 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
429 FileSystem fs = services.getMasterFileSystem().getFileSystem();
430 Path path = ref.write(fs, p);
431 assertTrue(fs.exists(path));
432 assertFalse(janitor.cleanParent(parent, r));
433
434 assertTrue(fs.delete(p, true));
435 assertTrue(janitor.cleanParent(parent, r));
436 } finally {
437 server.stop("shutdown");
438 }
439 }
440
441
442
443
444
445
446 @Test
447 public void testParentCleanedEvenIfDaughterGoneFirst()
448 throws IOException, InterruptedException {
449 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
450 "testParentCleanedEvenIfDaughterGoneFirst", Bytes.toBytes("eee"));
451 }
452
453
454
455
456
457
458 @Test
459 public void testLastParentCleanedEvenIfDaughterGoneFirst()
460 throws IOException, InterruptedException {
461 parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
462 "testLastParentCleanedEvenIfDaughterGoneFirst", new byte[0]);
463 }
464
465
466
467
468
469
470
471
472
473 private void parentWithSpecifiedEndKeyCleanedEvenIfDaughterGoneFirst(
474 final String rootDir, final byte[] lastEndKey)
475 throws IOException, InterruptedException {
476 HBaseTestingUtility htu = new HBaseTestingUtility();
477 setRootDirAndCleanIt(htu, rootDir);
478 Server server = new MockServer(htu);
479 MasterServices services = new MockMasterServices(server);
480 CatalogJanitor janitor = new CatalogJanitor(server, services);
481 final HTableDescriptor htd = createHTableDescriptor();
482
483
484
485
486 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
487 lastEndKey);
488
489
490 Thread.sleep(1001);
491
492
493 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
494 Bytes.toBytes("ccc"));
495 Thread.sleep(1001);
496
497 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
498 Bytes.toBytes("bbb"));
499 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
500 Bytes.toBytes("ccc"));
501
502
503 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
504 lastEndKey);
505 Thread.sleep(1001);
506
507 HRegionInfo splitba = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
508 Bytes.toBytes("ddd"));
509 HRegionInfo splitbb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ddd"),
510 lastEndKey);
511
512
513
514 SortedMap<HRegionInfo, Result> regions =
515 new TreeMap<HRegionInfo, Result>(new CatalogJanitor.SplitParentFirstComparator());
516
517 regions.put(parent, createResult(parent, splita, splitb));
518 regions.put(splitb, createResult(splitb, splitba, splitbb));
519 regions.put(splita, createResult(splita, splitaa, splitab));
520
521 int index = 0;
522 for (Map.Entry<HRegionInfo, Result> e: regions.entrySet()) {
523 if (index == 0) {
524 assertTrue(e.getKey().getEncodedName().equals(parent.getEncodedName()));
525 } else if (index == 1) {
526 assertTrue(e.getKey().getEncodedName().equals(splita.getEncodedName()));
527 } else if (index == 2) {
528 assertTrue(e.getKey().getEncodedName().equals(splitb.getEncodedName()));
529 }
530 index++;
531 }
532
533
534
535 Path splitaRef =
536 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
537
538 assertFalse(janitor.cleanParent(parent, regions.get(parent)));
539
540
541
542 assertTrue(janitor.cleanParent(splitb, regions.get(splitb)));
543
544
545
546
547
548 FileSystem fs = FileSystem.get(htu.getConfiguration());
549 assertTrue(fs.delete(splitaRef, true));
550
551 Path splitaaRef =
552 createReferences(services, htd, splita, splitaa, Bytes.toBytes("bbb"), false);
553 Path splitabRef =
554 createReferences(services, htd, splita, splitab, Bytes.toBytes("bbb"), true);
555
556
557 assertFalse(janitor.cleanParent(splita, regions.get(splita)));
558
559
560 assertTrue(fs.delete(splitaaRef, true));
561 assertTrue(fs.delete(splitabRef, true));
562 assertTrue(janitor.cleanParent(splita, regions.get(splita)));
563
564
565 assertTrue(janitor.cleanParent(parent, regions.get(parent)));
566
567 services.stop("test finished");
568 janitor.join();
569 }
570
571
572
573
574
575
576 @Test
577 public void testScanDoesNotCleanRegionsWithExistingParents() throws Exception {
578 HBaseTestingUtility htu = new HBaseTestingUtility();
579 setRootDirAndCleanIt(htu, "testScanDoesNotCleanRegionsWithExistingParents");
580 Server server = new MockServer(htu);
581 MasterServices services = new MockMasterServices(server);
582
583 final HTableDescriptor htd = createHTableDescriptor();
584
585
586
587
588 HRegionInfo parent = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
589 new byte[0], true);
590
591
592 Thread.sleep(1001);
593
594
595 HRegionInfo splita = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
596 Bytes.toBytes("ccc"), true);
597 Thread.sleep(1001);
598
599 HRegionInfo splitaa = new HRegionInfo(htd.getTableName(), Bytes.toBytes("aaa"),
600 Bytes.toBytes("bbb"), false);
601 HRegionInfo splitab = new HRegionInfo(htd.getTableName(), Bytes.toBytes("bbb"),
602 Bytes.toBytes("ccc"), false);
603
604
605 HRegionInfo splitb = new HRegionInfo(htd.getTableName(), Bytes.toBytes("ccc"),
606 new byte[0]);
607 Thread.sleep(1001);
608
609 final Map<HRegionInfo, Result> splitParents =
610 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
611 splitParents.put(parent, createResult(parent, splita, splitb));
612 splita.setOffline(true);
613 splitParents.put(splita, createResult(splita, splitaa,splitab));
614
615 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
616 CatalogJanitor janitor = spy(new CatalogJanitor(server, services));
617 doReturn(new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
618 10, mergedRegions, splitParents)).when(janitor)
619 .getMergedRegionsAndSplitParents();
620
621
622 Path splitaRef =
623 createReferences(services, htd, parent, splita, Bytes.toBytes("ccc"), false);
624
625
626 assertEquals(0, janitor.scan());
627
628
629 FileSystem fs = FileSystem.get(htu.getConfiguration());
630 assertTrue(fs.delete(splitaRef, true));
631
632
633 assertEquals(2, janitor.scan());
634
635 services.stop("test finished");
636 janitor.join();
637 }
638
639
640
641
642
643 @Test
644 public void testSplitParentFirstComparator() {
645 SplitParentFirstComparator comp = new SplitParentFirstComparator();
646 final HTableDescriptor htd = createHTableDescriptor();
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667 HRegionInfo rootRegion = new HRegionInfo(htd.getTableName(),
668 HConstants.EMPTY_START_ROW,
669 HConstants.EMPTY_END_ROW, true);
670 HRegionInfo firstRegion = new HRegionInfo(htd.getTableName(),
671 HConstants.EMPTY_START_ROW,
672 Bytes.toBytes("bbb"), true);
673 HRegionInfo lastRegion = new HRegionInfo(htd.getTableName(),
674 Bytes.toBytes("bbb"),
675 HConstants.EMPTY_END_ROW, true);
676
677 assertTrue(comp.compare(rootRegion, rootRegion) == 0);
678 assertTrue(comp.compare(firstRegion, firstRegion) == 0);
679 assertTrue(comp.compare(lastRegion, lastRegion) == 0);
680 assertTrue(comp.compare(rootRegion, firstRegion) < 0);
681 assertTrue(comp.compare(rootRegion, lastRegion) < 0);
682 assertTrue(comp.compare(firstRegion, lastRegion) < 0);
683
684
685 HRegionInfo firstRegiona = new HRegionInfo(htd.getTableName(),
686 HConstants.EMPTY_START_ROW,
687 Bytes.toBytes("aaa"), true);
688 HRegionInfo firstRegionb = new HRegionInfo(htd.getTableName(),
689 Bytes.toBytes("aaa"),
690 Bytes.toBytes("bbb"), true);
691
692 HRegionInfo lastRegiona = new HRegionInfo(htd.getTableName(),
693 Bytes.toBytes("bbb"),
694 Bytes.toBytes("ddd"), true);
695 HRegionInfo lastRegionb = new HRegionInfo(htd.getTableName(),
696 Bytes.toBytes("ddd"),
697 HConstants.EMPTY_END_ROW, true);
698
699 assertTrue(comp.compare(firstRegiona, firstRegiona) == 0);
700 assertTrue(comp.compare(firstRegionb, firstRegionb) == 0);
701 assertTrue(comp.compare(rootRegion, firstRegiona) < 0);
702 assertTrue(comp.compare(rootRegion, firstRegionb) < 0);
703 assertTrue(comp.compare(firstRegion, firstRegiona) < 0);
704 assertTrue(comp.compare(firstRegion, firstRegionb) < 0);
705 assertTrue(comp.compare(firstRegiona, firstRegionb) < 0);
706
707 assertTrue(comp.compare(lastRegiona, lastRegiona) == 0);
708 assertTrue(comp.compare(lastRegionb, lastRegionb) == 0);
709 assertTrue(comp.compare(rootRegion, lastRegiona) < 0);
710 assertTrue(comp.compare(rootRegion, lastRegionb) < 0);
711 assertTrue(comp.compare(lastRegion, lastRegiona) < 0);
712 assertTrue(comp.compare(lastRegion, lastRegionb) < 0);
713 assertTrue(comp.compare(lastRegiona, lastRegionb) < 0);
714
715 assertTrue(comp.compare(firstRegiona, lastRegiona) < 0);
716 assertTrue(comp.compare(firstRegiona, lastRegionb) < 0);
717 assertTrue(comp.compare(firstRegionb, lastRegiona) < 0);
718 assertTrue(comp.compare(firstRegionb, lastRegionb) < 0);
719
720 HRegionInfo lastRegionaa = new HRegionInfo(htd.getTableName(),
721 Bytes.toBytes("bbb"),
722 Bytes.toBytes("ccc"), false);
723 HRegionInfo lastRegionab = new HRegionInfo(htd.getTableName(),
724 Bytes.toBytes("ccc"),
725 Bytes.toBytes("ddd"), false);
726
727 assertTrue(comp.compare(lastRegiona, lastRegionaa) < 0);
728 assertTrue(comp.compare(lastRegiona, lastRegionab) < 0);
729 assertTrue(comp.compare(lastRegionaa, lastRegionab) < 0);
730
731 }
732
733 @Test
734 public void testArchiveOldRegion() throws Exception {
735 String table = "table";
736 HBaseTestingUtility htu = new HBaseTestingUtility();
737 setRootDirAndCleanIt(htu, "testCleanParent");
738 Server server = new MockServer(htu);
739 MasterServices services = new MockMasterServices(server);
740
741
742 CatalogJanitor janitor = new CatalogJanitor(server, services);
743
744
745 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
746 htd.addFamily(new HColumnDescriptor("f"));
747 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
748 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
749 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
750 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
751 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
752 Bytes.toBytes("ccc"),
753 Bytes.toBytes("eee"));
754
755
756
757 Result parentMetaRow = createResult(parent, splita, splitb);
758 FileSystem fs = FileSystem.get(htu.getConfiguration());
759 Path rootdir = services.getMasterFileSystem().getRootDir();
760
761
762
763 FSUtils.setRootDir(fs.getConf(), rootdir);
764 Path tabledir = FSUtils.getTableDir(rootdir, htd.getTableName());
765 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
766 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
767 tabledir, htd.getColumnFamilies()[0].getName());
768 LOG.debug("Table dir:" + tabledir);
769 LOG.debug("Store dir:" + storedir);
770 LOG.debug("Store archive dir:" + storeArchive);
771
772
773 FileStatus[] mockFiles = addMockStoreFiles(2, services, storedir);
774
775 FileStatus[] storeFiles = fs.listStatus(storedir);
776 int index = 0;
777 for (FileStatus file : storeFiles) {
778 LOG.debug("Have store file:" + file.getPath());
779 assertEquals("Got unexpected store file", mockFiles[index].getPath(),
780 storeFiles[index].getPath());
781 index++;
782 }
783
784
785 assertTrue(janitor.cleanParent(parent, parentMetaRow));
786 LOG.debug("Finished cleanup of parent region");
787
788
789 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
790 logFiles("archived files", storeFiles);
791 logFiles("archived files", archivedStoreFiles);
792
793 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
794
795
796 FSUtils.delete(fs, rootdir, true);
797 services.stop("Test finished");
798 server.stop("Test finished");
799 janitor.join();
800 }
801
802
803
804
805
806 private void logFiles(String description, FileStatus[] storeFiles) {
807 LOG.debug("Current " + description + ": ");
808 for (FileStatus file : storeFiles) {
809 LOG.debug(file.getPath());
810 }
811 }
812
813
814
815
816
817 @Test
818 public void testDuplicateHFileResolution() throws Exception {
819 String table = "table";
820 HBaseTestingUtility htu = new HBaseTestingUtility();
821 setRootDirAndCleanIt(htu, "testCleanParent");
822 Server server = new MockServer(htu);
823 MasterServices services = new MockMasterServices(server);
824
825
826 CatalogJanitor janitor = new CatalogJanitor(server, services);
827
828
829 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
830 htd.addFamily(new HColumnDescriptor("f"));
831 HRegionInfo parent = new HRegionInfo(htd.getTableName(),
832 Bytes.toBytes("aaa"), Bytes.toBytes("eee"));
833 HRegionInfo splita = new HRegionInfo(htd.getTableName(),
834 Bytes.toBytes("aaa"), Bytes.toBytes("ccc"));
835 HRegionInfo splitb = new HRegionInfo(htd.getTableName(),
836 Bytes.toBytes("ccc"), Bytes.toBytes("eee"));
837
838
839 Result r = createResult(parent, splita, splitb);
840
841 FileSystem fs = FileSystem.get(htu.getConfiguration());
842
843 Path rootdir = services.getMasterFileSystem().getRootDir();
844
845
846
847 FSUtils.setRootDir(fs.getConf(), rootdir);
848 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
849 Path storedir = HStore.getStoreHomedir(tabledir, parent, htd.getColumnFamilies()[0].getName());
850 System.out.println("Old root:" + rootdir);
851 System.out.println("Old table:" + tabledir);
852 System.out.println("Old store:" + storedir);
853
854 Path storeArchive = HFileArchiveUtil.getStoreArchivePath(services.getConfiguration(), parent,
855 tabledir, htd.getColumnFamilies()[0].getName());
856 System.out.println("Old archive:" + storeArchive);
857
858
859 addMockStoreFiles(2, services, storedir);
860
861 FileStatus[] storeFiles = fs.listStatus(storedir);
862
863 assertTrue(janitor.cleanParent(parent, r));
864
865
866 FileStatus[] archivedStoreFiles = fs.listStatus(storeArchive);
867 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs);
868
869
870
871 addMockStoreFiles(2, services, storedir);
872
873
874 assertTrue(janitor.cleanParent(parent, r));
875
876
877 archivedStoreFiles = fs.listStatus(storeArchive);
878 assertArchiveEqualToOriginal(storeFiles, archivedStoreFiles, fs, true);
879
880
881 services.stop("Test finished");
882 server.stop("shutdown");
883 janitor.join();
884 }
885
886 private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir)
887 throws IOException {
888
889 FileSystem fs = services.getMasterFileSystem().getFileSystem();
890 fs.mkdirs(storedir);
891
892 for (int i = 0; i < count; i++) {
893 Path storeFile = new Path(storedir, "_store" + i);
894 FSDataOutputStream dos = fs.create(storeFile, true);
895 dos.writeBytes("Some data: " + i);
896 dos.close();
897 }
898 LOG.debug("Adding " + count + " store files to the storedir:" + storedir);
899
900 FileStatus[] storeFiles = fs.listStatus(storedir);
901 assertEquals("Didn't have expected store files", count, storeFiles.length);
902 return storeFiles;
903 }
904
905 private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
906 final String subdir)
907 throws IOException {
908 Path testdir = htu.getDataTestDir(subdir);
909 FileSystem fs = FileSystem.get(htu.getConfiguration());
910 if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
911 FSUtils.setRootDir(htu.getConfiguration(), testdir);
912 return FSUtils.getRootDir(htu.getConfiguration()).toString();
913 }
914
915
916
917
918
919
920
921
922
923
924
925 private Path createReferences(final MasterServices services,
926 final HTableDescriptor htd, final HRegionInfo parent,
927 final HRegionInfo daughter, final byte [] midkey, final boolean top)
928 throws IOException {
929 Path rootdir = services.getMasterFileSystem().getRootDir();
930 Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable());
931 Path storedir = HStore.getStoreHomedir(tabledir, daughter,
932 htd.getColumnFamilies()[0].getName());
933 Reference ref =
934 top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey);
935 long now = System.currentTimeMillis();
936
937 Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName());
938 FileSystem fs = services.getMasterFileSystem().getFileSystem();
939 ref.write(fs, p);
940 return p;
941 }
942
943 private Result createResult(final HRegionInfo parent, final HRegionInfo a,
944 final HRegionInfo b)
945 throws IOException {
946 return MetaMockingUtil.getMetaTableRowResult(parent, null, a, b);
947 }
948
949 private HTableDescriptor createHTableDescriptor() {
950 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("t"));
951 htd.addFamily(new HColumnDescriptor("f"));
952 return htd;
953 }
954
955 }
956