View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.testclassification.LargeTests;
34  import org.apache.hadoop.hbase.TableNotFoundException;
35  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
37  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
38  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
39  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
40  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.FSUtils;
43  import org.junit.After;
44  import org.junit.AfterClass;
45  import org.junit.Before;
46  import org.junit.BeforeClass;
47  import org.junit.Test;
48  import org.junit.experimental.categories.Category;
49  
50  import com.google.common.collect.Lists;
51  
52  /**
53   * Test create/using/deleting snapshots from the client
54   * <p>
55   * This is an end-to-end test for the snapshot utility
56   */
57  @Category(LargeTests.class)
58  public class TestSnapshotFromClient {
59    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final int NUM_RS = 2;
62    private static final String STRING_TABLE_NAME = "test";
63    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
64    private static final TableName TABLE_NAME =
65        TableName.valueOf(STRING_TABLE_NAME);
66  
67    /**
68     * Setup the config for the cluster
69     * @throws Exception on failure
70     */
71    @BeforeClass
72    public static void setupCluster() throws Exception {
73      setupConf(UTIL.getConfiguration());
74      UTIL.startMiniCluster(NUM_RS);
75    }
76  
77    private static void setupConf(Configuration conf) {
78      // disable the ui
79      conf.setInt("hbase.regionsever.info.port", -1);
80      // change the flush size to a small amount, regulating number of store files
81      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
82      // so make sure we get a compaction when doing a load, but keep around some
83      // files in the store
84      conf.setInt("hbase.hstore.compaction.min", 10);
85      conf.setInt("hbase.hstore.compactionThreshold", 10);
86      // block writes if we get to 12 store files
87      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
88      // Enable snapshot
89      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
90      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91        ConstantSizeRegionSplitPolicy.class.getName());
92    }
93  
94    @Before
95    public void setup() throws Exception {
96      UTIL.createTable(TABLE_NAME, TEST_FAM);
97    }
98  
99    @After
100   public void tearDown() throws Exception {
101     UTIL.deleteTable(TABLE_NAME);
102     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
103     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
104   }
105 
106   @AfterClass
107   public static void cleanupTest() throws Exception {
108     try {
109       UTIL.shutdownMiniCluster();
110     } catch (Exception e) {
111       LOG.warn("failure shutting down cluster", e);
112     }
113   }
114 
115   /**
116    * Test snapshotting not allowed hbase:meta and -ROOT-
117    * @throws Exception
118    */
119   @Test (timeout=300000)
120   public void testMetaTablesSnapshot() throws Exception {
121     HBaseAdmin admin = UTIL.getHBaseAdmin();
122     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
123 
124     try {
125       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
126       fail("taking a snapshot of hbase:meta should not be allowed");
127     } catch (IllegalArgumentException e) {
128       // expected
129     }
130   }
131 
132   /**
133    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
134    *
135    * @throws Exception
136    */
137   @Test (timeout=300000)
138   public void testSnapshotDeletionWithRegex() throws Exception {
139     HBaseAdmin admin = UTIL.getHBaseAdmin();
140     // make sure we don't fail on listing snapshots
141     SnapshotTestingUtils.assertNoSnapshots(admin);
142 
143     // put some stuff in the table
144     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
145     UTIL.loadTable(table, TEST_FAM);
146     table.close();
147 
148     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
149     admin.snapshot(snapshot1, TABLE_NAME);
150     LOG.debug("Snapshot1 completed.");
151 
152     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
153     admin.snapshot(snapshot2, TABLE_NAME);
154     LOG.debug("Snapshot2 completed.");
155 
156     String snapshot3 = "3rdTableSnapshot";
157     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
158     LOG.debug(snapshot3 + " completed.");
159 
160     // delete the first two snapshots
161     admin.deleteSnapshots("TableSnapshot.*");
162     List<SnapshotDescription> snapshots = admin.listSnapshots();
163     assertEquals(1, snapshots.size());
164     assertEquals(snapshots.get(0).getName(), snapshot3);
165 
166     admin.deleteSnapshot(snapshot3);
167     admin.close();
168   }
169   /**
170    * Test snapshotting a table that is offline
171    * @throws Exception
172    */
173   @Test (timeout=300000)
174   public void testOfflineTableSnapshot() throws Exception {
175     HBaseAdmin admin = UTIL.getHBaseAdmin();
176     // make sure we don't fail on listing snapshots
177     SnapshotTestingUtils.assertNoSnapshots(admin);
178 
179     // put some stuff in the table
180     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
181     UTIL.loadTable(table, TEST_FAM, false);
182 
183     LOG.debug("FS state before disable:");
184     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
185       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
186     // XXX if this is flakey, might want to consider using the async version and looping as
187     // disableTable can succeed and still timeout.
188     admin.disableTable(TABLE_NAME);
189 
190     LOG.debug("FS state before snapshot:");
191     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
192       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
193 
194     // take a snapshot of the disabled table
195     final String SNAPSHOT_NAME = "offlineTableSnapshot";
196     byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
197 
198     SnapshotDescription desc = SnapshotDescription.newBuilder()
199       .setType(SnapshotDescription.Type.DISABLED)
200       .setTable(STRING_TABLE_NAME)
201       .setName(SNAPSHOT_NAME)
202       .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
203       .build();
204     admin.snapshot(desc);
205     LOG.debug("Snapshot completed.");
206 
207     // make sure we have the snapshot
208     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
209       snapshot, TABLE_NAME);
210 
211     // make sure its a valid snapshot
212     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
213     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
214     LOG.debug("FS state after snapshot:");
215     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
216       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
217 
218     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
219       admin, fs);
220 
221     admin.deleteSnapshot(snapshot);
222     snapshots = admin.listSnapshots();
223     SnapshotTestingUtils.assertNoSnapshots(admin);
224   }
225 
226   @Test (timeout=300000)
227   public void testSnapshotFailsOnNonExistantTable() throws Exception {
228     HBaseAdmin admin = UTIL.getHBaseAdmin();
229     // make sure we don't fail on listing snapshots
230     SnapshotTestingUtils.assertNoSnapshots(admin);
231     String tableName = "_not_a_table";
232 
233     // make sure the table doesn't exist
234     boolean fail = false;
235     do {
236     try {
237       admin.getTableDescriptor(Bytes.toBytes(tableName));
238       fail = true;
239           LOG.error("Table:" + tableName + " already exists, checking a new name");
240       tableName = tableName+"!";
241     } catch (TableNotFoundException e) {
242       fail = false;
243       }
244     } while (fail);
245 
246     // snapshot the non-existant table
247     try {
248       admin.snapshot("fail", tableName);
249       fail("Snapshot succeeded even though there is not table.");
250     } catch (SnapshotCreationException e) {
251       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
252     }
253   }
254 
255   @Test (timeout=300000)
256   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
257     // test with an empty table with one region
258 
259     HBaseAdmin admin = UTIL.getHBaseAdmin();
260     // make sure we don't fail on listing snapshots
261     SnapshotTestingUtils.assertNoSnapshots(admin);
262 
263     LOG.debug("FS state before disable:");
264     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
265       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
266     admin.disableTable(TABLE_NAME);
267 
268     LOG.debug("FS state before snapshot:");
269     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
270       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
271 
272     // take a snapshot of the disabled table
273     byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
274     admin.snapshot(snapshot, TABLE_NAME);
275     LOG.debug("Snapshot completed.");
276 
277     // make sure we have the snapshot
278     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
279       snapshot, TABLE_NAME);
280 
281     // make sure its a valid snapshot
282     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
283     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
284     LOG.debug("FS state after snapshot:");
285     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
286       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
287 
288     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
289     List<byte[]> nonEmptyCfs = Lists.newArrayList();
290     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
291       rootDir, admin, fs);
292 
293     admin.deleteSnapshot(snapshot);
294     snapshots = admin.listSnapshots();
295     SnapshotTestingUtils.assertNoSnapshots(admin);
296   }
297 }