1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.Before;
43 import org.junit.BeforeClass;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50 @Category(LargeTests.class)
51 public class TestSnapshotCloneIndependence {
52 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
53
54 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55
56 private static final int NUM_RS = 2;
57 private static final String STRING_TABLE_NAME = "test";
58 private static final String TEST_FAM_STR = "fam";
59 private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
60 private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
61
62
63
64
65
66 @BeforeClass
67 public static void setupCluster() throws Exception {
68 setupConf(UTIL.getConfiguration());
69 UTIL.startMiniCluster(NUM_RS);
70 }
71
72 private static void setupConf(Configuration conf) {
73
74 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
75
76 conf.setInt("hbase.regionsever.info.port", -1);
77
78 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
79
80
81 conf.setInt("hbase.hstore.compaction.min", 10);
82 conf.setInt("hbase.hstore.compactionThreshold", 10);
83
84 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
85 conf.setInt("hbase.regionserver.msginterval", 100);
86 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
87
88 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
89 ConstantSizeRegionSplitPolicy.class.getName());
90 }
91
92 @Before
93 public void setup() throws Exception {
94 UTIL.createTable(TABLE_NAME, TEST_FAM);
95 }
96
97 @After
98 public void tearDown() throws Exception {
99 UTIL.deleteTable(TABLE_NAME);
100 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
101 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
102 }
103
104 @AfterClass
105 public static void cleanupTest() throws Exception {
106 try {
107 UTIL.shutdownMiniCluster();
108 } catch (Exception e) {
109 LOG.warn("failure shutting down cluster", e);
110 }
111 }
112
113
114
115
116
117 @Test (timeout=300000)
118 public void testOnlineSnapshotAppendIndependent() throws Exception {
119 runTestSnapshotAppendIndependent(true);
120 }
121
122
123
124
125
126 @Test (timeout=300000)
127 public void testOfflineSnapshotAppendIndependent() throws Exception {
128 runTestSnapshotAppendIndependent(false);
129 }
130
131
132
133
134
135 @Test (timeout=300000)
136 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
137 runTestSnapshotMetadataChangesIndependent(true);
138 }
139
140
141
142
143
144 @Test (timeout=300000)
145 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
146 runTestSnapshotMetadataChangesIndependent(false);
147 }
148
149
150
151
152
153 @Test (timeout=300000)
154 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
155 runTestRegionOperationsIndependent(false);
156 }
157
158
159
160
161
162 @Test (timeout=300000)
163 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
164 runTestRegionOperationsIndependent(true);
165 }
166
167 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
168 for (int i = 0; i < 200; i++) {
169 try {
170 Thread.sleep(50);
171 } catch (InterruptedException e) {
172
173 Thread.currentThread().interrupt();
174 }
175 if (t.getRegionLocations().size() > originalCount) {
176 return;
177 }
178 }
179 throw new Exception("Split did not increase the number of regions");
180 }
181
182
183
184
185
186
187 private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
188 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
189 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
190
191 HBaseAdmin admin = UTIL.getHBaseAdmin();
192 final long startTime = System.currentTimeMillis();
193 final TableName localTableName =
194 TableName.valueOf(STRING_TABLE_NAME + startTime);
195
196 HTable original = UTIL.createTable(localTableName, TEST_FAM);
197 try {
198
199 UTIL.loadTable(original, TEST_FAM);
200 final int origTableRowCount = UTIL.countRows(original);
201
202
203 final String snapshotNameAsString = "snapshot_" + localTableName;
204 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
205
206 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
207 snapshotNameAsString, rootDir, fs, online);
208
209 if (!online) {
210 admin.enableTable(localTableName);
211 }
212 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
213 admin.cloneSnapshot(snapshotName, cloneTableName);
214
215 HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
216
217 try {
218 final int clonedTableRowCount = UTIL.countRows(clonedTable);
219
220 Assert.assertEquals(
221 "The line counts of original and cloned tables do not match after clone. ",
222 origTableRowCount, clonedTableRowCount);
223
224
225 final String rowKey = "new-row-" + System.currentTimeMillis();
226
227 Put p = new Put(Bytes.toBytes(rowKey));
228 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
229 original.put(p);
230 original.flushCommits();
231
232
233 Assert.assertEquals("The row count of the original table was not modified by the put",
234 origTableRowCount + 1, UTIL.countRows(original));
235 Assert.assertEquals(
236 "The row count of the cloned table changed as a result of addition to the original",
237 clonedTableRowCount, UTIL.countRows(clonedTable));
238
239 p = new Put(Bytes.toBytes(rowKey));
240 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
241 clonedTable.put(p);
242 clonedTable.flushCommits();
243
244
245 Assert.assertEquals(
246 "The row count of the original table was modified by the put to the clone",
247 origTableRowCount + 1, UTIL.countRows(original));
248 Assert.assertEquals("The row count of the cloned table was not modified by the put",
249 clonedTableRowCount + 1, UTIL.countRows(clonedTable));
250 } finally {
251
252 clonedTable.close();
253 }
254 } finally {
255
256 original.close();
257 }
258 }
259
260
261
262
263
264 private void runTestRegionOperationsIndependent(boolean online) throws Exception {
265 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
266 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
267
268
269 HBaseAdmin admin = UTIL.getHBaseAdmin();
270 final long startTime = System.currentTimeMillis();
271 final TableName localTableName =
272 TableName.valueOf(STRING_TABLE_NAME + startTime);
273 HTable original = UTIL.createTable(localTableName, TEST_FAM);
274 UTIL.loadTable(original, TEST_FAM);
275 final int loadedTableCount = UTIL.countRows(original);
276 System.out.println("Original table has: " + loadedTableCount + " rows");
277
278 final String snapshotNameAsString = "snapshot_" + localTableName;
279
280
281 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
282 snapshotNameAsString, rootDir, fs, online);
283
284 if (!online) {
285 admin.enableTable(localTableName);
286 }
287
288 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
289
290
291 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
292 admin.cloneSnapshot(snapshotName, cloneTableName);
293
294
295 original.clearRegionCache();
296 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
297
298 final int originalRegionCount = originalTableHRegions.size();
299 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
300 Assert.assertEquals(
301 "The number of regions in the cloned table is different than in the original table.",
302 originalRegionCount, cloneTableRegionCount);
303
304
305 admin.split(originalTableHRegions.get(0).getRegionName());
306 waitOnSplit(original, originalRegionCount);
307
308
309 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
310 Assert.assertEquals(
311 "The number of regions in the cloned table changed though none of its regions were split.",
312 cloneTableRegionCount, cloneTableRegionCount2);
313 }
314
315
316
317
318
319
320 private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
321 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
322 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
323
324
325 HBaseAdmin admin = UTIL.getHBaseAdmin();
326 final long startTime = System.currentTimeMillis();
327 final TableName localTableName =
328 TableName.valueOf(STRING_TABLE_NAME + startTime);
329 HTable original = UTIL.createTable(localTableName, TEST_FAM);
330 UTIL.loadTable(original, TEST_FAM);
331
332 final String snapshotNameAsString = "snapshot_" + localTableName;
333
334
335 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
336 snapshotNameAsString, rootDir, fs, online);
337
338 if (!online) {
339 admin.enableTable(localTableName);
340 }
341 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
342
343
344 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
345 admin.cloneSnapshot(snapshotName, cloneTableName);
346
347
348 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
349 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
350
351 admin.disableTable(localTableName);
352 admin.addColumn(localTableName, hcd);
353
354
355 admin.enableTable(localTableName);
356
357
358
359
360 HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
361 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
362
363 Assert.assertTrue("The original family was not found. There is something wrong. ",
364 originalTableDescriptor.hasFamily(TEST_FAM));
365 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
366 clonedTableDescriptor.hasFamily(TEST_FAM));
367
368 Assert.assertTrue("The new family was not found. ",
369 originalTableDescriptor.hasFamily(TEST_FAM_2));
370 Assert.assertTrue("The new family was not found. ",
371 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
372 }
373 }