View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.Arrays;
26  import java.util.Map;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.FSDataInputStream;
31  import org.apache.hadoop.fs.FSDataOutputStream;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestCase;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.Tag;
39  import org.apache.hadoop.hbase.io.compress.Compression;
40  import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41  import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42  import org.apache.hadoop.hbase.testclassification.SmallTests;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.io.Writable;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * test hfile features.
49   * <p>
50   * Copied from
51   * <a href="https://issues.apache.org/jira/browse/HADOOP-3315">hadoop-3315 tfile</a>.
52   * Remove after tfile is committed and use the tfile version of this class
53   * instead.</p>
54   */
55  @Category(SmallTests.class)
56  public class TestHFile extends HBaseTestCase {
57    static final Log LOG = LogFactory.getLog(TestHFile.class);
58  
59    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60    private static String ROOT_DIR =
61      TEST_UTIL.getDataTestDir("TestHFile").toString();
62    private final int minBlockSize = 512;
63    private static String localFormatter = "%010d";
64    private static CacheConfig cacheConf = null;
65    private Map<String, Long> startingMetrics;
66  
67    @Override
68    public void setUp() throws Exception {
69      super.setUp();
70    }
71  
72    @Override
73    public void tearDown() throws Exception {
74      super.tearDown();
75    }
76  
77  
78    /**
79     * Test empty HFile.
80     * Test all features work reasonably when hfile is empty of entries.
81     * @throws IOException
82     */
83    public void testEmptyHFile() throws IOException {
84      if (cacheConf == null) cacheConf = new CacheConfig(conf);
85      Path f = new Path(ROOT_DIR, getName());
86      HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
87      Writer w =
88          HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
89      w.close();
90      Reader r = HFile.createReader(fs, f, cacheConf, conf);
91      r.loadFileInfo();
92      assertNull(r.getFirstKey());
93      assertNull(r.getLastKey());
94    }
95  
96    /**
97     * Create 0-length hfile and show that it fails
98     */
99    public void testCorrupt0LengthHFile() throws IOException {
100     if (cacheConf == null) cacheConf = new CacheConfig(conf);
101     Path f = new Path(ROOT_DIR, getName());
102     FSDataOutputStream fsos = fs.create(f);
103     fsos.close();
104 
105     try {
106       Reader r = HFile.createReader(fs, f, cacheConf, conf);
107     } catch (CorruptHFileException che) {
108       // Expected failure
109       return;
110     }
111     fail("Should have thrown exception");
112   }
113 
114   public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
115     FileStatus fst = fs.getFileStatus(src);
116     long len = fst.getLen();
117     len = len / 2 ;
118 
119     // create a truncated hfile
120     FSDataOutputStream fdos = fs.create(dst);
121     byte[] buf = new byte[(int)len];
122     FSDataInputStream fdis = fs.open(src);
123     fdis.read(buf);
124     fdos.write(buf);
125     fdis.close();
126     fdos.close();
127   }
128 
129   /**
130    * Create a truncated hfile and verify that exception thrown.
131    */
132   public void testCorruptTruncatedHFile() throws IOException {
133     if (cacheConf == null) cacheConf = new CacheConfig(conf);
134     Path f = new Path(ROOT_DIR, getName());
135     HFileContext  context = new HFileContextBuilder().build();
136     Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
137         .withFileContext(context).create();
138     writeSomeRecords(w, 0, 100, false);
139     w.close();
140 
141     Path trunc = new Path(f.getParent(), "trucated");
142     truncateFile(fs, w.getPath(), trunc);
143 
144     try {
145       Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
146     } catch (CorruptHFileException che) {
147       // Expected failure
148       return;
149     }
150     fail("Should have thrown exception");
151   }
152 
153   // write some records into the tfile
154   // write them twice
155   private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
156       throws IOException {
157     String value = "value";
158     for (int i = start; i < (start + n); i++) {
159       String key = String.format(localFormatter, Integer.valueOf(i));
160       if (useTags) {
161         Tag t = new Tag((byte) 1, "myTag1");
162         writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key), t.getBuffer());
163       } else {
164         writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
165       }
166     }
167     return (start + n);
168   }
169 
170   private void readAllRecords(HFileScanner scanner) throws IOException {
171     readAndCheckbytes(scanner, 0, 100);
172   }
173 
174   // read the records and check
175   private int readAndCheckbytes(HFileScanner scanner, int start, int n)
176       throws IOException {
177     String value = "value";
178     int i = start;
179     for (; i < (start + n); i++) {
180       ByteBuffer key = scanner.getKey();
181       ByteBuffer val = scanner.getValue();
182       String keyStr = String.format(localFormatter, Integer.valueOf(i));
183       String valStr = value + keyStr;
184       byte [] keyBytes = Bytes.toBytes(key);
185       assertTrue("bytes for keys do not match " + keyStr + " " +
186         Bytes.toString(Bytes.toBytes(key)),
187           Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
188       byte [] valBytes = Bytes.toBytes(val);
189       assertTrue("bytes for vals do not match " + valStr + " " +
190         Bytes.toString(valBytes),
191         Arrays.equals(Bytes.toBytes(valStr), valBytes));
192       if (!scanner.next()) {
193         break;
194       }
195     }
196     assertEquals(i, start + n - 1);
197     return (start + n);
198   }
199 
200   private byte[] getSomeKey(int rowId) {
201     return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
202   }
203 
204   private void writeRecords(Writer writer, boolean useTags) throws IOException {
205     writeSomeRecords(writer, 0, 100, useTags);
206     writer.close();
207   }
208 
209   private FSDataOutputStream createFSOutput(Path name) throws IOException {
210     //if (fs.exists(name)) fs.delete(name, true);
211     FSDataOutputStream fout = fs.create(name);
212     return fout;
213   }
214 
215   /**
216    * test none codecs
217    * @param useTags 
218    */
219   void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
220     if (useTags) {
221       conf.setInt("hfile.format.version", 3);
222     }
223     if (cacheConf == null) cacheConf = new CacheConfig(conf);
224     Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
225     FSDataOutputStream fout = createFSOutput(ncTFile);
226     HFileContext meta = new HFileContextBuilder()
227                         .withBlockSize(minBlockSize)
228                         .withCompression(AbstractHFileWriter.compressionByName(codec))
229                         .build();
230     Writer writer = HFile.getWriterFactory(conf, cacheConf)
231         .withOutputStream(fout)
232         .withFileContext(meta)
233         // NOTE: This test is dependent on this deprecated nonstandard comparator
234         .withComparator(new KeyValue.RawBytesComparator())
235         .create();
236     LOG.info(writer);
237     writeRecords(writer, useTags);
238     fout.close();
239     FSDataInputStream fin = fs.open(ncTFile);
240     Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
241       fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
242     System.out.println(cacheConf.toString());
243     // Load up the index.
244     reader.loadFileInfo();
245     // Get a scanner that caches and that does not use pread.
246     HFileScanner scanner = reader.getScanner(true, false);
247     // Align scanner at start of the file.
248     scanner.seekTo();
249     readAllRecords(scanner);
250     scanner.seekTo(getSomeKey(50));
251     assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
252     // read the key and see if it matches
253     ByteBuffer readKey = scanner.getKey();
254     assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
255       Bytes.toBytes(readKey)));
256 
257     scanner.seekTo(new byte[0]);
258     ByteBuffer val1 = scanner.getValue();
259     scanner.seekTo(new byte[0]);
260     ByteBuffer val2 = scanner.getValue();
261     assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
262 
263     reader.close();
264     fin.close();
265     fs.delete(ncTFile, true);
266   }
267 
268   public void testTFileFeatures() throws IOException {
269     testTFilefeaturesInternals(false);
270     testTFilefeaturesInternals(true);
271   }
272 
273   protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
274     basicWithSomeCodec("none", useTags);
275     basicWithSomeCodec("gz", useTags);
276   }
277 
278   private void writeNumMetablocks(Writer writer, int n) {
279     for (int i = 0; i < n; i++) {
280       writer.appendMetaBlock("HFileMeta" + i, new Writable() {
281         private int val;
282         public Writable setVal(int val) { this.val = val; return this; }
283         
284         @Override
285         public void write(DataOutput out) throws IOException {
286           out.write(("something to test" + val).getBytes());
287         }
288         
289         @Override
290         public void readFields(DataInput in) throws IOException { }
291       }.setVal(i));
292     }
293   }
294 
295   private void someTestingWithMetaBlock(Writer writer) {
296     writeNumMetablocks(writer, 10);
297   }
298 
299   private void readNumMetablocks(Reader reader, int n) throws IOException {
300     for (int i = 0; i < n; i++) {
301       ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
302       ByteBuffer expected = 
303         ByteBuffer.wrap(("something to test" + i).getBytes());
304       assertEquals("failed to match metadata",
305         Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
306     }
307   }
308 
309   private void someReadingWithMetaBlock(Reader reader) throws IOException {
310     readNumMetablocks(reader, 10);
311   }
312 
313   private void metablocks(final String compress) throws Exception {
314     if (cacheConf == null) cacheConf = new CacheConfig(conf);
315     Path mFile = new Path(ROOT_DIR, "meta.hfile");
316     FSDataOutputStream fout = createFSOutput(mFile);
317     HFileContext meta = new HFileContextBuilder()
318                         .withCompression(AbstractHFileWriter.compressionByName(compress))
319                         .withBlockSize(minBlockSize).build();
320     Writer writer = HFile.getWriterFactory(conf, cacheConf)
321         .withOutputStream(fout)
322         .withFileContext(meta)
323         .create();
324     someTestingWithMetaBlock(writer);
325     writer.close();
326     fout.close();
327     FSDataInputStream fin = fs.open(mFile);
328     Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
329         this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
330     reader.loadFileInfo();
331     // No data -- this should return false.
332     assertFalse(reader.getScanner(false, false).seekTo());
333     someReadingWithMetaBlock(reader);
334     fs.delete(mFile, true);
335     reader.close();
336     fin.close();
337   }
338 
339   // test meta blocks for tfiles
340   public void testMetaBlocks() throws Exception {
341     metablocks("none");
342     metablocks("gz");
343   }
344 
345   public void testNullMetaBlocks() throws Exception {
346     if (cacheConf == null) cacheConf = new CacheConfig(conf);
347     for (Compression.Algorithm compressAlgo : 
348         HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
349       Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
350       FSDataOutputStream fout = createFSOutput(mFile);
351       HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
352                           .withBlockSize(minBlockSize).build();
353       Writer writer = HFile.getWriterFactory(conf, cacheConf)
354           .withOutputStream(fout)
355           .withFileContext(meta)
356           .create();
357       writer.append("foo".getBytes(), "value".getBytes());
358       writer.close();
359       fout.close();
360       Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
361       reader.loadFileInfo();
362       assertNull(reader.getMetaBlock("non-existant", false));
363     }
364   }
365 
366   /**
367    * Make sure the ordinals for our compression algorithms do not change on us.
368    */
369   public void testCompressionOrdinance() {
370     assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
371     assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
372     assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
373     assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
374     assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
375   }
376 
377 }
378