View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.ArrayList;
27  import java.util.Arrays;
28  import java.util.Collection;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Set;
32  import java.util.HashSet;
33  import java.util.TreeSet;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FSDataOutputStream;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.fs.PathFilter;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.TableNotEnabledException;
49  import org.apache.hadoop.hbase.Waiter;
50  import org.apache.hadoop.hbase.client.Durability;
51  import org.apache.hadoop.hbase.client.HBaseAdmin;
52  import org.apache.hadoop.hbase.client.HTable;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
55  import org.apache.hadoop.hbase.io.HFileLink;
56  import org.apache.hadoop.hbase.master.HMaster;
57  import org.apache.hadoop.hbase.master.MasterFileSystem;
58  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
59  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
61  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
62  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
63  import org.apache.hadoop.hbase.regionserver.HRegion;
64  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
65  import org.apache.hadoop.hbase.regionserver.HRegionServer;
66  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
67  import org.apache.hadoop.hbase.util.Bytes;
68  import org.apache.hadoop.hbase.util.FSTableDescriptors;
69  import org.apache.hadoop.hbase.util.FSUtils;
70  import org.apache.hadoop.hbase.util.FSVisitor;
71  import org.apache.hadoop.hbase.util.FSUtils;
72  import org.apache.hadoop.hbase.util.MD5Hash;
73  import org.junit.Assert;
74  
75  import com.google.protobuf.ServiceException;
76  
77  /**
78   * Utilities class for snapshots
79   */
80  public class SnapshotTestingUtils {
81  
82    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
83    private static byte[] KEYS = Bytes.toBytes("0123456789");
84  
85    /**
86     * Assert that we don't have any snapshots lists
87     *
88     * @throws IOException
89     *           if the admin operation fails
90     */
91    public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
92      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
93          .size());
94    }
95  
96    /**
97     * Make sure that there is only one snapshot returned from the master and its
98     * name and table match the passed in parameters.
99     */
100   public static List<SnapshotDescription> assertExistsMatchingSnapshot(
101       HBaseAdmin admin, String snapshotName, TableName tableName)
102       throws IOException {
103     // list the snapshot
104     List<SnapshotDescription> snapshots = admin.listSnapshots();
105 
106     List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
107     for (SnapshotDescription sd : snapshots) {
108       if (snapshotName.equals(sd.getName()) &&
109           tableName.equals(TableName.valueOf(sd.getTable()))) {
110         returnedSnapshots.add(sd);
111       }
112     }
113 
114     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
115     return returnedSnapshots;
116   }
117 
118   /**
119    * Make sure that there is only one snapshot returned from the master
120    */
121   public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
122       SnapshotDescription snapshot) throws IOException {
123     assertOneSnapshotThatMatches(admin, snapshot.getName(),
124         TableName.valueOf(snapshot.getTable()));
125   }
126 
127   /**
128    * Make sure that there is only one snapshot returned from the master and its
129    * name and table match the passed in parameters.
130    */
131   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
132       HBaseAdmin admin, String snapshotName, TableName tableName)
133       throws IOException {
134     // list the snapshot
135     List<SnapshotDescription> snapshots = admin.listSnapshots();
136 
137     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
138     assertEquals(snapshotName, snapshots.get(0).getName());
139     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
140 
141     return snapshots;
142   }
143 
144   /**
145    * Make sure that there is only one snapshot returned from the master and its
146    * name and table match the passed in parameters.
147    */
148   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
149       HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
150     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
151         tableName);
152   }
153 
154   /**
155    * Confirm that the snapshot contains references to all the files that should
156    * be in the snapshot.
157    */
158   public static void confirmSnapshotValid(
159       SnapshotDescription snapshotDescriptor, TableName tableName,
160       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
161       throws IOException {
162     ArrayList nonEmptyTestFamilies = new ArrayList(1);
163     nonEmptyTestFamilies.add(testFamily);
164     confirmSnapshotValid(snapshotDescriptor, tableName,
165       nonEmptyTestFamilies, null, rootDir, admin, fs);
166   }
167 
168   /**
169    * Confirm that the snapshot has no references files but only metadata.
170    */
171   public static void confirmEmptySnapshotValid(
172       SnapshotDescription snapshotDescriptor, TableName tableName,
173       byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
174       throws IOException {
175     ArrayList emptyTestFamilies = new ArrayList(1);
176     emptyTestFamilies.add(testFamily);
177     confirmSnapshotValid(snapshotDescriptor, tableName,
178       null, emptyTestFamilies, rootDir, admin, fs);
179   }
180 
181   /**
182    * Confirm that the snapshot contains references to all the files that should
183    * be in the snapshot. This method also perform some redundant check like
184    * the existence of the snapshotinfo or the regioninfo which are done always
185    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
186    */
187   public static void confirmSnapshotValid(
188       SnapshotDescription snapshotDescriptor, TableName tableName,
189       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
190       Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
191     final Configuration conf = admin.getConfiguration();
192 
193     // check snapshot dir
194     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
195         snapshotDescriptor, rootDir);
196     assertTrue(fs.exists(snapshotDir));
197 
198     SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
199 
200     // Extract regions and families with store files
201     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
202 
203     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
204     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
205     for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
206       SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
207           new SnapshotReferenceUtil.StoreFileVisitor() {
208         @Override
209         public void storeFile(final HRegionInfo regionInfo, final String family,
210               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
211           snapshotFamilies.add(Bytes.toBytes(family));
212         }
213       });
214     }
215 
216     // Verify that there are store files in the specified families
217     if (nonEmptyTestFamilies != null) {
218       for (final byte[] familyName: nonEmptyTestFamilies) {
219         assertTrue(snapshotFamilies.contains(familyName));
220       }
221     }
222 
223     // Verify that there are no store files in the specified families
224     if (emptyTestFamilies != null) {
225       for (final byte[] familyName: emptyTestFamilies) {
226         assertFalse(snapshotFamilies.contains(familyName));
227       }
228     }
229 
230     // check the region snapshot for all the regions
231     List<HRegionInfo> regions = admin.getTableRegions(tableName);
232     assertEquals(regions.size(), regionManifests.size());
233 
234     // Verify Regions (redundant check, see MasterSnapshotVerifier)
235     for (HRegionInfo info : regions) {
236       String regionName = info.getEncodedName();
237       assertTrue(regionManifests.containsKey(regionName));
238     }
239   }
240 
241   /**
242    * Helper method for testing async snapshot operations. Just waits for the
243    * given snapshot to complete on the server by repeatedly checking the master.
244    *
245    * @param master: the master running the snapshot
246    * @param snapshot: the snapshot to check
247    * @param sleep: amount to sleep between checks to see if the snapshot is done
248    * @throws ServiceException if the snapshot fails
249    */
250   public static void waitForSnapshotToComplete(HMaster master,
251       SnapshotDescription snapshot, long sleep) throws ServiceException {
252     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
253         .setSnapshot(snapshot).build();
254     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
255         .buildPartial();
256     while (!done.getDone()) {
257       done = master.isSnapshotDone(null, request);
258       try {
259         Thread.sleep(sleep);
260       } catch (InterruptedException e) {
261         throw new ServiceException(e);
262       }
263     }
264   }
265 
266   /*
267    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
268    * except for the last CorruptedSnapshotException
269    */
270   public static void snapshot(HBaseAdmin admin,
271       final String snapshotName, final String tableName,
272       SnapshotDescription.Type type, int numTries) throws IOException {
273     int tries = 0;
274     CorruptedSnapshotException lastEx = null;
275     while (tries++ < numTries) {
276       try {
277         admin.snapshot(snapshotName, tableName, type);
278         return;
279       } catch (CorruptedSnapshotException cse) {
280         LOG.warn("Got CorruptedSnapshotException", cse);
281         lastEx = cse;
282       }
283     }
284     throw lastEx;
285   }
286 
287   public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
288       throws IOException {
289     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
290   }
291 
292   public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
293       throws IOException {
294     // delete the taken snapshot
295     admin.deleteSnapshot(snapshotName);
296     assertNoSnapshots(admin);
297   }
298 
299   /**
300    * Expect the snapshot to throw an error when checking if the snapshot is
301    * complete
302    *
303    * @param master master to check
304    * @param snapshot the {@link SnapshotDescription} request to pass to the master
305    * @param clazz expected exception from the master
306    */
307   public static void expectSnapshotDoneException(HMaster master,
308       IsSnapshotDoneRequest snapshot,
309       Class<? extends HBaseSnapshotException> clazz) {
310     try {
311       master.isSnapshotDone(null, snapshot);
312       Assert.fail("didn't fail to lookup a snapshot");
313     } catch (ServiceException se) {
314       try {
315         throw ProtobufUtil.getRemoteException(se);
316       } catch (HBaseSnapshotException e) {
317         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
318       } catch (Throwable t) {
319         Assert.fail("Threw an unexpected exception:" + t);
320       }
321     }
322   }
323 
324   /**
325    * List all the HFiles in the given table
326    *
327    * @param fs: FileSystem where the table lives
328    * @param tableDir directory of the table
329    * @return array of the current HFiles in the table (could be a zero-length array)
330    * @throws IOException on unexecpted error reading the FS
331    */
332   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
333       throws IOException {
334     final ArrayList<Path> hfiles = new ArrayList<Path>();
335     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
336       @Override
337       public void storeFile(final String region, final String family, final String hfileName)
338           throws IOException {
339         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
340       }
341     });
342     return hfiles.toArray(new Path[hfiles.size()]);
343   }
344 
345   public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
346       throws IOException {
347     Path[] files = listHFiles(fs, tableDir);
348     String[] names = new String[files.length];
349     for (int i = 0; i < files.length; ++i) {
350       names[i] = files[i].getName();
351     }
352     Arrays.sort(names);
353     return names;
354   }
355 
356   /**
357    * Take a snapshot of the specified table and verify that the given family is
358    * not empty. Note that this will leave the table disabled
359    * in the case of an offline snapshot.
360    */
361   public static void createSnapshotAndValidate(HBaseAdmin admin,
362       TableName tableName, String familyName, String snapshotNameString,
363       Path rootDir, FileSystem fs, boolean onlineSnapshot)
364       throws Exception {
365     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
366     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
367     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
368                               snapshotNameString, rootDir, fs, onlineSnapshot);
369   }
370 
371   /**
372    * Take a snapshot of the specified table and verify the given families.
373    * Note that this will leave the table disabled in the case of an offline snapshot.
374    */
375   public static void createSnapshotAndValidate(HBaseAdmin admin,
376       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
377       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
378         throws Exception {
379     if (!onlineSnapshot) {
380       try {
381         admin.disableTable(tableName);
382       } catch (TableNotEnabledException tne) {
383         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
384             "already disabled.");
385       }
386     }
387     admin.snapshot(snapshotNameString, tableName);
388 
389     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
390       snapshotNameString, tableName);
391     if (snapshots == null || snapshots.size() != 1) {
392       Assert.fail("Incorrect number of snapshots for table " + tableName);
393     }
394 
395     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
396       emptyFamilyNames, rootDir, admin, fs);
397   }
398 
399   /**
400    * Corrupt the specified snapshot by deleting some files.
401    *
402    * @param util {@link HBaseTestingUtility}
403    * @param snapshotName name of the snapshot to corrupt
404    * @return array of the corrupted HFiles
405    * @throws IOException on unexecpted error reading the FS
406    */
407   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
408       throws IOException {
409     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
410     final FileSystem fs = mfs.getFileSystem();
411 
412     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
413                                                                         mfs.getRootDir());
414     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
415     final TableName table = TableName.valueOf(snapshotDesc.getTable());
416 
417     final ArrayList corruptedFiles = new ArrayList();
418     final Configuration conf = util.getConfiguration();
419     SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
420         new SnapshotReferenceUtil.StoreFileVisitor() {
421       @Override
422       public void storeFile(final HRegionInfo regionInfo, final String family,
423             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
424         String region = regionInfo.getEncodedName();
425         String hfile = storeFile.getName();
426         HFileLink link = HFileLink.create(conf, table, region, family, hfile);
427         if (corruptedFiles.size() % 2 == 0) {
428           fs.delete(link.getAvailablePath(fs));
429           corruptedFiles.add(hfile);
430         }
431       }
432     });
433 
434     assertTrue(corruptedFiles.size() > 0);
435     return corruptedFiles;
436   }
437 
438   // ==========================================================================
439   //  Snapshot Mock
440   // ==========================================================================
441   public static class SnapshotMock {
442     private final static String TEST_FAMILY = "cf";
443     public final static int TEST_NUM_REGIONS = 4;
444 
445     private final Configuration conf;
446     private final FileSystem fs;
447     private final Path rootDir;
448 
449     static class RegionData {
450       public HRegionInfo hri;
451       public Path tableDir;
452       public Path[] files;
453 
454       public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
455         this.tableDir = tableDir;
456         this.hri = hri;
457         this.files = new Path[nfiles];
458       }
459     }
460 
461     public static class SnapshotBuilder {
462       private final RegionData[] tableRegions;
463       private final SnapshotDescription desc;
464       private final HTableDescriptor htd;
465       private final Configuration conf;
466       private final FileSystem fs;
467       private final Path rootDir;
468       private Path snapshotDir;
469       private int snapshotted = 0;
470 
471       public SnapshotBuilder(final Configuration conf, final FileSystem fs,
472           final Path rootDir, final HTableDescriptor htd,
473           final SnapshotDescription desc, final RegionData[] tableRegions)
474           throws IOException {
475         this.fs = fs;
476         this.conf = conf;
477         this.rootDir = rootDir;
478         this.htd = htd;
479         this.desc = desc;
480         this.tableRegions = tableRegions;
481         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
482         new FSTableDescriptors(conf)
483           .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
484       }
485 
486       public HTableDescriptor getTableDescriptor() {
487         return this.htd;
488       }
489 
490       public SnapshotDescription getSnapshotDescription() {
491         return this.desc;
492       }
493 
494       public Path getSnapshotsDir() {
495         return this.snapshotDir;
496       }
497 
498       public Path[] addRegion() throws IOException {
499         return addRegion(desc);
500       }
501 
502       public Path[] addRegionV1() throws IOException {
503         return addRegion(desc.toBuilder()
504                           .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
505                           .build());
506       }
507 
508       public Path[] addRegionV2() throws IOException {
509         return addRegion(desc.toBuilder()
510                           .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
511                           .build());
512       }
513 
514       private Path[] addRegion(final SnapshotDescription desc) throws IOException {
515         if (this.snapshotted == tableRegions.length) {
516           throw new UnsupportedOperationException("No more regions in the table");
517         }
518 
519         RegionData regionData = tableRegions[this.snapshotted++];
520         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
521         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
522         manifest.addRegion(regionData.tableDir, regionData.hri);
523         return regionData.files;
524       }
525 
526       public Path commit() throws IOException {
527         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
528         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
529         manifest.addTableDescriptor(htd);
530         manifest.consolidate();
531         SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
532         snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
533         return snapshotDir;
534       }
535     }
536 
537     public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
538       this.fs = fs;
539       this.conf = conf;
540       this.rootDir = rootDir;
541     }
542 
543     public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
544       return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
545     }
546 
547     public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
548       return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
549     }
550 
551     private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
552         throws IOException {
553       HTableDescriptor htd = createHtd(snapshotName);
554       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
555 
556       RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
557 
558       SnapshotDescription desc = SnapshotDescription.newBuilder()
559         .setTable(htd.getNameAsString())
560         .setName(snapshotName)
561         .setVersion(version)
562         .build();
563 
564       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
565       SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
566       return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
567     }
568 
569     public HTableDescriptor createHtd(final String tableName) {
570       HTableDescriptor htd = new HTableDescriptor(tableName);
571       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
572       return htd;
573     }
574 
575     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
576         throws IOException {
577       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
578       new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
579 
580       assertTrue(nregions % 2 == 0);
581       RegionData[] regions = new RegionData[nregions];
582       for (int i = 0; i < regions.length; i += 2) {
583         byte[] startKey = Bytes.toBytes(0 + i * 2);
584         byte[] endKey = Bytes.toBytes(1 + i * 2);
585 
586         // First region, simple with one plain hfile.
587         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
588         HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
589         regions[i] = new RegionData(tableDir, hri, 3);
590         for (int j = 0; j < regions[i].files.length; ++j) {
591           Path storeFile = createStoreFile(rfs.createTempName());
592           regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
593         }
594 
595         // Second region, used to test the split case.
596         // This region contains a reference to the hfile in the first region.
597         startKey = Bytes.toBytes(2 + i * 2);
598         endKey = Bytes.toBytes(3 + i * 2);
599         hri = new HRegionInfo(htd.getTableName());
600         rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
601         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
602         for (int j = 0; j < regions[i].files.length; ++j) {
603           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
604           Path refFile = createStoreFile(new Path(rootDir, refName));
605           regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
606         }
607       }
608       return regions;
609     }
610 
611     private Path createStoreFile(final Path storeFile)
612         throws IOException {
613       FSDataOutputStream out = fs.create(storeFile);
614       try {
615         out.write(Bytes.toBytes(storeFile.toString()));
616       } finally {
617         out.close();
618       }
619       return storeFile;
620     }
621   }
622 
623   // ==========================================================================
624   //  Table Helpers
625   // ==========================================================================
626   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
627                                             final TableName tableName)
628       throws IOException, InterruptedException {
629     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
630     List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
631     for (HRegion region : onlineRegions) {
632       region.waitForFlushesAndCompactions();
633     }
634     // Wait up to 60 seconds for a table to be available.
635     final HBaseAdmin hBaseAdmin = util.getHBaseAdmin();
636     util.waitFor(60000, new Waiter.Predicate<IOException>() {
637       @Override
638       public boolean evaluate() throws IOException {
639         return hBaseAdmin.isTableAvailable(tableName);
640       }
641     });
642   }
643 
644   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
645       final byte[]... families) throws IOException, InterruptedException {
646     HTableDescriptor htd = new HTableDescriptor(tableName);
647     for (byte[] family : families) {
648       HColumnDescriptor hcd = new HColumnDescriptor(family);
649       htd.addFamily(hcd);
650     }
651     byte[][] splitKeys = new byte[KEYS.length-2][];
652     for (int i = 0; i < splitKeys.length; ++i) {
653       splitKeys[i] = new byte[] { KEYS[i+1] };
654     }
655     util.createTable(htd, splitKeys);
656     assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
657   }
658 
659   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
660       byte[]... families) throws IOException, InterruptedException {
661     loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
662   }
663 
664   public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
665       byte[]... families) throws IOException, InterruptedException {
666     table.setAutoFlush(false, true);
667 
668     // Ensure one row per region
669     assertTrue(rows >= KEYS.length);
670     for (byte k0: KEYS) {
671       byte[] k = new byte[] { k0 };
672       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
673       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
674       putData(table, families, key, value);
675       rows--;
676     }
677 
678     // Add other extra rows. more rows, more files
679     while (rows-- > 0) {
680       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
681       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
682       putData(table, families, key, value);
683     }
684     table.flushCommits();
685 
686     waitForTableToBeOnline(util, table.getName());
687   }
688 
689   private static void putData(final HTable table, final byte[][] families,
690       final byte[] key, final byte[] value) throws IOException {
691     byte[] q = Bytes.toBytes("q");
692     Put put = new Put(key);
693     put.setDurability(Durability.SKIP_WAL);
694     for (byte[] family: families) {
695       put.add(family, q, value);
696     }
697     table.put(put);
698   }
699 
700   public static void deleteAllSnapshots(final HBaseAdmin admin)
701       throws IOException {
702     // Delete all the snapshots
703     for (SnapshotDescription snapshot: admin.listSnapshots()) {
704       admin.deleteSnapshot(snapshot.getName());
705     }
706     SnapshotTestingUtils.assertNoSnapshots(admin);
707   }
708 
709   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
710       throws IOException {
711     // Ensure the archiver to be empty
712     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
713     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
714     mfs.getFileSystem().delete(archiveDir, true);
715   }
716 
717   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
718       long expectedRows) throws IOException {
719     HTable table = new HTable(util.getConfiguration(), tableName);
720     try {
721       assertEquals(expectedRows, util.countRows(table));
722     } finally {
723       table.close();
724     }
725   }
726 }