View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.CountDownLatch;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.testclassification.LargeTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.TableNotFoundException;
43  import org.apache.hadoop.hbase.client.HBaseAdmin;
44  import org.apache.hadoop.hbase.client.HTable;
45  import org.apache.hadoop.hbase.master.HMaster;
46  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
47  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
48  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.FSUtils;
51  import org.junit.After;
52  import org.junit.AfterClass;
53  import org.junit.Before;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  import org.junit.experimental.categories.Category;
57  
58  /**
59   * Test creating/using/deleting snapshots from the client
60   * <p>
61   * This is an end-to-end test for the snapshot utility
62   *
63   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
64   * because there will be a few more flavors of snapshots that need to run these tests.
65   */
66  @Category(LargeTests.class)
67  public class TestFlushSnapshotFromClient {
68    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
69    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
70    private static final int NUM_RS = 2;
71    private static final String STRING_TABLE_NAME = "test";
72    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
74    private static final TableName TABLE_NAME =
75        TableName.valueOf(STRING_TABLE_NAME);
76    private final int DEFAULT_NUM_ROWS = 100;
77  
78    /**
79     * Setup the config for the cluster
80     * @throws Exception on failure
81     */
82    @BeforeClass
83    public static void setupCluster() throws Exception {
84      // Uncomment the following lines if more verbosity is needed for
85      // debugging (see HBASE-12285 for details).
86      //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
87      //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
88      //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
89      setupConf(UTIL.getConfiguration());
90      UTIL.startMiniCluster(NUM_RS);
91    }
92  
93    private static void setupConf(Configuration conf) {
94      // disable the ui
95      conf.setInt("hbase.regionsever.info.port", -1);
96      // change the flush size to a small amount, regulating number of store files
97      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
98      // so make sure we get a compaction when doing a load, but keep around some
99      // files in the store
100     conf.setInt("hbase.hstore.compaction.min", 10);
101     conf.setInt("hbase.hstore.compactionThreshold", 10);
102     // block writes if we get to 12 store files
103     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
104     // Enable snapshot
105     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
106     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
107       ConstantSizeRegionSplitPolicy.class.getName());
108   }
109 
110   @Before
111   public void setup() throws Exception {
112     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
113   }
114 
115   @After
116   public void tearDown() throws Exception {
117     UTIL.deleteTable(TABLE_NAME);
118 
119     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
120     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
121   }
122 
123   @AfterClass
124   public static void cleanupTest() throws Exception {
125     try {
126       UTIL.shutdownMiniCluster();
127     } catch (Exception e) {
128       LOG.warn("failure shutting down cluster", e);
129     }
130   }
131 
132   /**
133    * Test simple flush snapshotting a table that is online
134    * @throws Exception
135    */
136   @Test (timeout=300000)
137   public void testFlushTableSnapshot() throws Exception {
138     HBaseAdmin admin = UTIL.getHBaseAdmin();
139     // make sure we don't fail on listing snapshots
140     SnapshotTestingUtils.assertNoSnapshots(admin);
141 
142     // put some stuff in the table
143     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
144     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
145 
146     LOG.debug("FS state before snapshot:");
147     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
148         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
149 
150     // take a snapshot of the enabled table
151     String snapshotString = "offlineTableSnapshot";
152     byte[] snapshot = Bytes.toBytes(snapshotString);
153     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
154     LOG.debug("Snapshot completed.");
155 
156     // make sure we have the snapshot
157     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
158       snapshot, TABLE_NAME);
159 
160     // make sure its a valid snapshot
161     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
162     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
163     LOG.debug("FS state after snapshot:");
164     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
165         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
166 
167     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
168         admin, fs);
169   }
170 
171    /**
172    * Test snapshotting a table that is online without flushing
173    * @throws Exception
174    */
175   @Test
176   public void testSkipFlushTableSnapshot() throws Exception {
177     HBaseAdmin admin = UTIL.getHBaseAdmin();
178     // make sure we don't fail on listing snapshots
179     SnapshotTestingUtils.assertNoSnapshots(admin);
180 
181     // put some stuff in the table
182     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
183     UTIL.loadTable(table, TEST_FAM);
184 
185     LOG.debug("FS state before snapshot:");
186     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
187         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
188 
189     // take a snapshot of the enabled table
190     String snapshotString = "skipFlushTableSnapshot";
191     byte[] snapshot = Bytes.toBytes(snapshotString);
192     admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
193     LOG.debug("Snapshot completed.");
194 
195     // make sure we have the snapshot
196     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
197         snapshot, TABLE_NAME);
198 
199     // make sure its a valid snapshot
200     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
201     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
202     LOG.debug("FS state after snapshot:");
203     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
204         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
205 
206     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
207         admin, fs);
208 
209     admin.deleteSnapshot(snapshot);
210     snapshots = admin.listSnapshots();
211     SnapshotTestingUtils.assertNoSnapshots(admin);
212   }
213 
214 
215   /**
216    * Test simple flush snapshotting a table that is online
217    * @throws Exception
218    */
219   @Test (timeout=300000)
220   public void testFlushTableSnapshotWithProcedure() throws Exception {
221     HBaseAdmin admin = UTIL.getHBaseAdmin();
222     // make sure we don't fail on listing snapshots
223     SnapshotTestingUtils.assertNoSnapshots(admin);
224 
225     // put some stuff in the table
226     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
227     SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
228 
229     LOG.debug("FS state before snapshot:");
230     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
231         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
232 
233     // take a snapshot of the enabled table
234     String snapshotString = "offlineTableSnapshot";
235     byte[] snapshot = Bytes.toBytes(snapshotString);
236     Map<String, String> props = new HashMap<String, String>();
237     props.put("table", STRING_TABLE_NAME);
238     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
239         snapshotString, props);
240 
241 
242     LOG.debug("Snapshot completed.");
243 
244     // make sure we have the snapshot
245     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
246       snapshot, TABLE_NAME);
247 
248     // make sure its a valid snapshot
249     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
250     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
251     LOG.debug("FS state after snapshot:");
252     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
253         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
254 
255     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
256         admin, fs);
257   }
258 
259   @Test (timeout=300000)
260   public void testSnapshotFailsOnNonExistantTable() throws Exception {
261     HBaseAdmin admin = UTIL.getHBaseAdmin();
262     // make sure we don't fail on listing snapshots
263     SnapshotTestingUtils.assertNoSnapshots(admin);
264     String tableName = "_not_a_table";
265 
266     // make sure the table doesn't exist
267     boolean fail = false;
268     do {
269     try {
270       admin.getTableDescriptor(Bytes.toBytes(tableName));
271       fail = true;
272       LOG.error("Table:" + tableName + " already exists, checking a new name");
273       tableName = tableName+"!";
274     } catch (TableNotFoundException e) {
275       fail = false;
276       }
277     } while (fail);
278 
279     // snapshot the non-existant table
280     try {
281       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
282       fail("Snapshot succeeded even though there is not table.");
283     } catch (SnapshotCreationException e) {
284       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
285     }
286   }
287 
288   @Test(timeout = 300000)
289   public void testAsyncFlushSnapshot() throws Exception {
290     HBaseAdmin admin = UTIL.getHBaseAdmin();
291     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
292         .setTable(TABLE_NAME.getNameAsString())
293         .setType(SnapshotDescription.Type.FLUSH)
294         .build();
295 
296     // take the snapshot async
297     admin.takeSnapshotAsync(snapshot);
298 
299     // constantly loop, looking for the snapshot to complete
300     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
301     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
302     LOG.info(" === Async Snapshot Completed ===");
303     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
304       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
305     // make sure we get the snapshot
306     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
307   }
308 
309   @Test (timeout=300000)
310   public void testSnapshotStateAfterMerge() throws Exception {
311     int numRows = DEFAULT_NUM_ROWS;
312     HBaseAdmin admin = UTIL.getHBaseAdmin();
313     // make sure we don't fail on listing snapshots
314     SnapshotTestingUtils.assertNoSnapshots(admin);
315     // load the table so we have some data
316     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
317 
318     // Take a snapshot
319     String snapshotBeforeMergeName = "snapshotBeforeMerge";
320     admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
321 
322     // Clone the table
323     String cloneBeforeMergeName = "cloneBeforeMerge";
324     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
325     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
326 
327     // Merge two regions
328     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
329     Collections.sort(regions, new Comparator<HRegionInfo>() {
330       public int compare(HRegionInfo r1, HRegionInfo r2) {
331         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
332       }
333     });
334 
335     int numRegions = admin.getTableRegions(TABLE_NAME).size();
336     int numRegionsAfterMerge = numRegions - 2;
337     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
338         regions.get(2).getEncodedNameAsBytes(), true);
339     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
340         regions.get(6).getEncodedNameAsBytes(), true);
341 
342     // Verify that there's one region less
343     waitRegionsAfterMerge(numRegionsAfterMerge);
344     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
345 
346     // Clone the table
347     String cloneAfterMergeName = "cloneAfterMerge";
348     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
349     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
350 
351     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
352     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
353     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
354 
355     // test that we can delete the snapshot
356     UTIL.deleteTable(cloneAfterMergeName);
357     UTIL.deleteTable(cloneBeforeMergeName);
358   }
359 
360   @Test (timeout=300000)
361   public void testTakeSnapshotAfterMerge() throws Exception {
362     int numRows = DEFAULT_NUM_ROWS;
363     HBaseAdmin admin = UTIL.getHBaseAdmin();
364     // make sure we don't fail on listing snapshots
365     SnapshotTestingUtils.assertNoSnapshots(admin);
366     // load the table so we have some data
367     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
368 
369     // Merge two regions
370     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
371     Collections.sort(regions, new Comparator<HRegionInfo>() {
372       public int compare(HRegionInfo r1, HRegionInfo r2) {
373         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
374       }
375     });
376 
377     int numRegions = admin.getTableRegions(TABLE_NAME).size();
378     int numRegionsAfterMerge = numRegions - 2;
379     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
380         regions.get(2).getEncodedNameAsBytes(), true);
381     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
382         regions.get(6).getEncodedNameAsBytes(), true);
383 
384     waitRegionsAfterMerge(numRegionsAfterMerge);
385     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
386 
387     // Take a snapshot
388     String snapshotName = "snapshotAfterMerge";
389     SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
390       SnapshotDescription.Type.FLUSH, 3);
391 
392     // Clone the table
393     String cloneName = "cloneMerge";
394     admin.cloneSnapshot(snapshotName, cloneName);
395     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
396 
397     SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
398     SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
399 
400     // test that we can delete the snapshot
401     UTIL.deleteTable(cloneName);
402   }
403 
404   /**
405    * Basic end-to-end test of simple-flush-based snapshots
406    */
407   @Test (timeout=300000)
408   public void testFlushCreateListDestroy() throws Exception {
409     LOG.debug("------- Starting Snapshot test -------------");
410     HBaseAdmin admin = UTIL.getHBaseAdmin();
411     // make sure we don't fail on listing snapshots
412     SnapshotTestingUtils.assertNoSnapshots(admin);
413     // load the table so we have some data
414     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
415 
416     String snapshotName = "flushSnapshotCreateListDestroy";
417     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
418     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
419     SnapshotTestingUtils.createSnapshotAndValidate(admin,
420       TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
421       snapshotName, rootDir, fs, true);
422   }
423 
424   /**
425    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
426    * same table currently running and that concurrent snapshots on different tables can both
427    * succeed concurretly.
428    */
429   @Test(timeout=300000)
430   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
431     final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
432     final TableName TABLE2_NAME =
433         TableName.valueOf(STRING_TABLE2_NAME);
434 
435     int ssNum = 20;
436     HBaseAdmin admin = UTIL.getHBaseAdmin();
437     // make sure we don't fail on listing snapshots
438     SnapshotTestingUtils.assertNoSnapshots(admin);
439     // create second testing table
440     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
441     // load the table so we have some data
442     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
443     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
444 
445     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
446     // We'll have one of these per thread
447     class SSRunnable implements Runnable {
448       SnapshotDescription ss;
449       SSRunnable(SnapshotDescription ss) {
450         this.ss = ss;
451       }
452 
453       @Override
454       public void run() {
455         try {
456           HBaseAdmin admin = UTIL.getHBaseAdmin();
457           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
458           admin.takeSnapshotAsync(ss);
459         } catch (Exception e) {
460           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
461               ss)
462               + ".  This is ok, we expect some", e);
463         }
464         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
465         toBeSubmitted.countDown();
466       }
467     };
468 
469     // build descriptions
470     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
471     for (int i = 0; i < ssNum; i++) {
472       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
473       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
474       builder.setName("ss"+i);
475       builder.setType(SnapshotDescription.Type.FLUSH);
476       descs[i] = builder.build();
477     }
478 
479     // kick each off its own thread
480     for (int i=0 ; i < ssNum; i++) {
481       new Thread(new SSRunnable(descs[i])).start();
482     }
483 
484     // wait until all have been submitted
485     toBeSubmitted.await();
486 
487     // loop until all are done.
488     while (true) {
489       int doneCount = 0;
490       for (SnapshotDescription ss : descs) {
491         try {
492           if (admin.isSnapshotFinished(ss)) {
493             doneCount++;
494           }
495         } catch (Exception e) {
496           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
497           doneCount++;
498         }
499       }
500       if (doneCount == descs.length) {
501         break;
502       }
503       Thread.sleep(100);
504     }
505 
506     // dump for debugging
507     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
508 
509     List<SnapshotDescription> taken = admin.listSnapshots();
510     int takenSize = taken.size();
511     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
512     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
513         " issued many requests", takenSize < ssNum && takenSize > 0);
514 
515     // Verify that there's at least one snapshot per table
516     int t1SnapshotsCount = 0;
517     int t2SnapshotsCount = 0;
518     for (SnapshotDescription ss : taken) {
519       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
520         t1SnapshotsCount++;
521       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
522         t2SnapshotsCount++;
523       }
524     }
525     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
526     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
527 
528     UTIL.deleteTable(TABLE2_NAME);
529   }
530 
531   private void logFSTree(Path root) throws IOException {
532     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
533   }
534 
535   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
536       throws IOException, InterruptedException {
537     HBaseAdmin admin = UTIL.getHBaseAdmin();
538     // Verify that there's one region less
539     long startTime = System.currentTimeMillis();
540     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
541       // This may be flaky... if after 15sec the merge is not complete give up
542       // it will fail in the assertEquals(numRegionsAfterMerge).
543       if ((System.currentTimeMillis() - startTime) > 15000)
544         break;
545       Thread.sleep(100);
546     }
547     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
548   }
549 }