1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.concurrent.CountDownLatch;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.testclassification.LargeTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.TableNotFoundException;
43 import org.apache.hadoop.hbase.client.HBaseAdmin;
44 import org.apache.hadoop.hbase.client.HTable;
45 import org.apache.hadoop.hbase.master.HMaster;
46 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
47 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
48 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.junit.After;
52 import org.junit.AfterClass;
53 import org.junit.Before;
54 import org.junit.BeforeClass;
55 import org.junit.Test;
56 import org.junit.experimental.categories.Category;
57
58
59
60
61
62
63
64
65
66 @Category(LargeTests.class)
67 public class TestFlushSnapshotFromClient {
68 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
69 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
70 private static final int NUM_RS = 2;
71 private static final String STRING_TABLE_NAME = "test";
72 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73 private static final byte[] TEST_QUAL = Bytes.toBytes("q");
74 private static final TableName TABLE_NAME =
75 TableName.valueOf(STRING_TABLE_NAME);
76 private final int DEFAULT_NUM_ROWS = 100;
77
78
79
80
81
82 @BeforeClass
83 public static void setupCluster() throws Exception {
84
85
86
87
88
89 setupConf(UTIL.getConfiguration());
90 UTIL.startMiniCluster(NUM_RS);
91 }
92
93 private static void setupConf(Configuration conf) {
94
95 conf.setInt("hbase.regionsever.info.port", -1);
96
97 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
98
99
100 conf.setInt("hbase.hstore.compaction.min", 10);
101 conf.setInt("hbase.hstore.compactionThreshold", 10);
102
103 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
104
105 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
106 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
107 ConstantSizeRegionSplitPolicy.class.getName());
108 }
109
110 @Before
111 public void setup() throws Exception {
112 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
113 }
114
115 @After
116 public void tearDown() throws Exception {
117 UTIL.deleteTable(TABLE_NAME);
118
119 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
120 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
121 }
122
123 @AfterClass
124 public static void cleanupTest() throws Exception {
125 try {
126 UTIL.shutdownMiniCluster();
127 } catch (Exception e) {
128 LOG.warn("failure shutting down cluster", e);
129 }
130 }
131
132
133
134
135
136 @Test (timeout=300000)
137 public void testFlushTableSnapshot() throws Exception {
138 HBaseAdmin admin = UTIL.getHBaseAdmin();
139
140 SnapshotTestingUtils.assertNoSnapshots(admin);
141
142
143 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
144 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
145
146 LOG.debug("FS state before snapshot:");
147 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
148 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
149
150
151 String snapshotString = "offlineTableSnapshot";
152 byte[] snapshot = Bytes.toBytes(snapshotString);
153 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
154 LOG.debug("Snapshot completed.");
155
156
157 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
158 snapshot, TABLE_NAME);
159
160
161 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
162 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
163 LOG.debug("FS state after snapshot:");
164 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
165 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
166
167 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
168 admin, fs);
169 }
170
171
172
173
174
175 @Test
176 public void testSkipFlushTableSnapshot() throws Exception {
177 HBaseAdmin admin = UTIL.getHBaseAdmin();
178
179 SnapshotTestingUtils.assertNoSnapshots(admin);
180
181
182 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
183 UTIL.loadTable(table, TEST_FAM);
184
185 LOG.debug("FS state before snapshot:");
186 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
187 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
188
189
190 String snapshotString = "skipFlushTableSnapshot";
191 byte[] snapshot = Bytes.toBytes(snapshotString);
192 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
193 LOG.debug("Snapshot completed.");
194
195
196 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
197 snapshot, TABLE_NAME);
198
199
200 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
201 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
202 LOG.debug("FS state after snapshot:");
203 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
204 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
205
206 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
207 admin, fs);
208
209 admin.deleteSnapshot(snapshot);
210 snapshots = admin.listSnapshots();
211 SnapshotTestingUtils.assertNoSnapshots(admin);
212 }
213
214
215
216
217
218
219 @Test (timeout=300000)
220 public void testFlushTableSnapshotWithProcedure() throws Exception {
221 HBaseAdmin admin = UTIL.getHBaseAdmin();
222
223 SnapshotTestingUtils.assertNoSnapshots(admin);
224
225
226 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
227 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
228
229 LOG.debug("FS state before snapshot:");
230 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
231 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
232
233
234 String snapshotString = "offlineTableSnapshot";
235 byte[] snapshot = Bytes.toBytes(snapshotString);
236 Map<String, String> props = new HashMap<String, String>();
237 props.put("table", STRING_TABLE_NAME);
238 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
239 snapshotString, props);
240
241
242 LOG.debug("Snapshot completed.");
243
244
245 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
246 snapshot, TABLE_NAME);
247
248
249 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
250 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
251 LOG.debug("FS state after snapshot:");
252 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
253 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
254
255 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
256 admin, fs);
257 }
258
259 @Test (timeout=300000)
260 public void testSnapshotFailsOnNonExistantTable() throws Exception {
261 HBaseAdmin admin = UTIL.getHBaseAdmin();
262
263 SnapshotTestingUtils.assertNoSnapshots(admin);
264 String tableName = "_not_a_table";
265
266
267 boolean fail = false;
268 do {
269 try {
270 admin.getTableDescriptor(Bytes.toBytes(tableName));
271 fail = true;
272 LOG.error("Table:" + tableName + " already exists, checking a new name");
273 tableName = tableName+"!";
274 } catch (TableNotFoundException e) {
275 fail = false;
276 }
277 } while (fail);
278
279
280 try {
281 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
282 fail("Snapshot succeeded even though there is not table.");
283 } catch (SnapshotCreationException e) {
284 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
285 }
286 }
287
288 @Test(timeout = 300000)
289 public void testAsyncFlushSnapshot() throws Exception {
290 HBaseAdmin admin = UTIL.getHBaseAdmin();
291 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
292 .setTable(TABLE_NAME.getNameAsString())
293 .setType(SnapshotDescription.Type.FLUSH)
294 .build();
295
296
297 admin.takeSnapshotAsync(snapshot);
298
299
300 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
301 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
302 LOG.info(" === Async Snapshot Completed ===");
303 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
304 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
305
306 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
307 }
308
309 @Test (timeout=300000)
310 public void testSnapshotStateAfterMerge() throws Exception {
311 int numRows = DEFAULT_NUM_ROWS;
312 HBaseAdmin admin = UTIL.getHBaseAdmin();
313
314 SnapshotTestingUtils.assertNoSnapshots(admin);
315
316 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
317
318
319 String snapshotBeforeMergeName = "snapshotBeforeMerge";
320 admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
321
322
323 String cloneBeforeMergeName = "cloneBeforeMerge";
324 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
325 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
326
327
328 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
329 Collections.sort(regions, new Comparator<HRegionInfo>() {
330 public int compare(HRegionInfo r1, HRegionInfo r2) {
331 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
332 }
333 });
334
335 int numRegions = admin.getTableRegions(TABLE_NAME).size();
336 int numRegionsAfterMerge = numRegions - 2;
337 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
338 regions.get(2).getEncodedNameAsBytes(), true);
339 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
340 regions.get(6).getEncodedNameAsBytes(), true);
341
342
343 waitRegionsAfterMerge(numRegionsAfterMerge);
344 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
345
346
347 String cloneAfterMergeName = "cloneAfterMerge";
348 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
349 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
350
351 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
352 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
353 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
354
355
356 UTIL.deleteTable(cloneAfterMergeName);
357 UTIL.deleteTable(cloneBeforeMergeName);
358 }
359
360 @Test (timeout=300000)
361 public void testTakeSnapshotAfterMerge() throws Exception {
362 int numRows = DEFAULT_NUM_ROWS;
363 HBaseAdmin admin = UTIL.getHBaseAdmin();
364
365 SnapshotTestingUtils.assertNoSnapshots(admin);
366
367 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
368
369
370 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
371 Collections.sort(regions, new Comparator<HRegionInfo>() {
372 public int compare(HRegionInfo r1, HRegionInfo r2) {
373 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
374 }
375 });
376
377 int numRegions = admin.getTableRegions(TABLE_NAME).size();
378 int numRegionsAfterMerge = numRegions - 2;
379 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
380 regions.get(2).getEncodedNameAsBytes(), true);
381 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
382 regions.get(6).getEncodedNameAsBytes(), true);
383
384 waitRegionsAfterMerge(numRegionsAfterMerge);
385 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
386
387
388 String snapshotName = "snapshotAfterMerge";
389 SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
390 SnapshotDescription.Type.FLUSH, 3);
391
392
393 String cloneName = "cloneMerge";
394 admin.cloneSnapshot(snapshotName, cloneName);
395 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
396
397 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
398 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
399
400
401 UTIL.deleteTable(cloneName);
402 }
403
404
405
406
407 @Test (timeout=300000)
408 public void testFlushCreateListDestroy() throws Exception {
409 LOG.debug("------- Starting Snapshot test -------------");
410 HBaseAdmin admin = UTIL.getHBaseAdmin();
411
412 SnapshotTestingUtils.assertNoSnapshots(admin);
413
414 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
415
416 String snapshotName = "flushSnapshotCreateListDestroy";
417 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
418 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
419 SnapshotTestingUtils.createSnapshotAndValidate(admin,
420 TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
421 snapshotName, rootDir, fs, true);
422 }
423
424
425
426
427
428
429 @Test(timeout=300000)
430 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
431 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
432 final TableName TABLE2_NAME =
433 TableName.valueOf(STRING_TABLE2_NAME);
434
435 int ssNum = 20;
436 HBaseAdmin admin = UTIL.getHBaseAdmin();
437
438 SnapshotTestingUtils.assertNoSnapshots(admin);
439
440 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
441
442 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
443 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
444
445 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
446
447 class SSRunnable implements Runnable {
448 SnapshotDescription ss;
449 SSRunnable(SnapshotDescription ss) {
450 this.ss = ss;
451 }
452
453 @Override
454 public void run() {
455 try {
456 HBaseAdmin admin = UTIL.getHBaseAdmin();
457 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
458 admin.takeSnapshotAsync(ss);
459 } catch (Exception e) {
460 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
461 ss)
462 + ". This is ok, we expect some", e);
463 }
464 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
465 toBeSubmitted.countDown();
466 }
467 };
468
469
470 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
471 for (int i = 0; i < ssNum; i++) {
472 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
473 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
474 builder.setName("ss"+i);
475 builder.setType(SnapshotDescription.Type.FLUSH);
476 descs[i] = builder.build();
477 }
478
479
480 for (int i=0 ; i < ssNum; i++) {
481 new Thread(new SSRunnable(descs[i])).start();
482 }
483
484
485 toBeSubmitted.await();
486
487
488 while (true) {
489 int doneCount = 0;
490 for (SnapshotDescription ss : descs) {
491 try {
492 if (admin.isSnapshotFinished(ss)) {
493 doneCount++;
494 }
495 } catch (Exception e) {
496 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
497 doneCount++;
498 }
499 }
500 if (doneCount == descs.length) {
501 break;
502 }
503 Thread.sleep(100);
504 }
505
506
507 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
508
509 List<SnapshotDescription> taken = admin.listSnapshots();
510 int takenSize = taken.size();
511 LOG.info("Taken " + takenSize + " snapshots: " + taken);
512 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
513 " issued many requests", takenSize < ssNum && takenSize > 0);
514
515
516 int t1SnapshotsCount = 0;
517 int t2SnapshotsCount = 0;
518 for (SnapshotDescription ss : taken) {
519 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
520 t1SnapshotsCount++;
521 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
522 t2SnapshotsCount++;
523 }
524 }
525 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
526 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
527
528 UTIL.deleteTable(TABLE2_NAME);
529 }
530
531 private void logFSTree(Path root) throws IOException {
532 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
533 }
534
535 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
536 throws IOException, InterruptedException {
537 HBaseAdmin admin = UTIL.getHBaseAdmin();
538
539 long startTime = System.currentTimeMillis();
540 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
541
542
543 if ((System.currentTimeMillis() - startTime) > 15000)
544 break;
545 Thread.sleep(100);
546 }
547 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
548 }
549 }