1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Arrays;
26 import java.util.Map;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.FSDataInputStream;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestCase;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.Tag;
39 import org.apache.hadoop.hbase.io.compress.Compression;
40 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42 import org.apache.hadoop.hbase.testclassification.SmallTests;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.io.Writable;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50
51
52
53
54
55 @Category(SmallTests.class)
56 public class TestHFile extends HBaseTestCase {
57 static final Log LOG = LogFactory.getLog(TestHFile.class);
58
59 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60 private static String ROOT_DIR =
61 TEST_UTIL.getDataTestDir("TestHFile").toString();
62 private final int minBlockSize = 512;
63 private static String localFormatter = "%010d";
64 private static CacheConfig cacheConf = null;
65 private Map<String, Long> startingMetrics;
66
67 @Override
68 public void setUp() throws Exception {
69 super.setUp();
70 }
71
72 @Override
73 public void tearDown() throws Exception {
74 super.tearDown();
75 }
76
77
78
79
80
81
82
83 public void testEmptyHFile() throws IOException {
84 if (cacheConf == null) cacheConf = new CacheConfig(conf);
85 Path f = new Path(ROOT_DIR, getName());
86 HFileContext context = new HFileContextBuilder().withIncludesTags(false).build();
87 Writer w =
88 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create();
89 w.close();
90 Reader r = HFile.createReader(fs, f, cacheConf, conf);
91 r.loadFileInfo();
92 assertNull(r.getFirstKey());
93 assertNull(r.getLastKey());
94 }
95
96
97
98
99 public void testCorrupt0LengthHFile() throws IOException {
100 if (cacheConf == null) cacheConf = new CacheConfig(conf);
101 Path f = new Path(ROOT_DIR, getName());
102 FSDataOutputStream fsos = fs.create(f);
103 fsos.close();
104
105 try {
106 Reader r = HFile.createReader(fs, f, cacheConf, conf);
107 } catch (CorruptHFileException che) {
108
109 return;
110 }
111 fail("Should have thrown exception");
112 }
113
114 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
115 FileStatus fst = fs.getFileStatus(src);
116 long len = fst.getLen();
117 len = len / 2 ;
118
119
120 FSDataOutputStream fdos = fs.create(dst);
121 byte[] buf = new byte[(int)len];
122 FSDataInputStream fdis = fs.open(src);
123 fdis.read(buf);
124 fdos.write(buf);
125 fdis.close();
126 fdos.close();
127 }
128
129
130
131
132 public void testCorruptTruncatedHFile() throws IOException {
133 if (cacheConf == null) cacheConf = new CacheConfig(conf);
134 Path f = new Path(ROOT_DIR, getName());
135 HFileContext context = new HFileContextBuilder().build();
136 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f)
137 .withFileContext(context).create();
138 writeSomeRecords(w, 0, 100, false);
139 w.close();
140
141 Path trunc = new Path(f.getParent(), "trucated");
142 truncateFile(fs, w.getPath(), trunc);
143
144 try {
145 Reader r = HFile.createReader(fs, trunc, cacheConf, conf);
146 } catch (CorruptHFileException che) {
147
148 return;
149 }
150 fail("Should have thrown exception");
151 }
152
153
154
155 private int writeSomeRecords(Writer writer, int start, int n, boolean useTags)
156 throws IOException {
157 String value = "value";
158 for (int i = start; i < (start + n); i++) {
159 String key = String.format(localFormatter, Integer.valueOf(i));
160 if (useTags) {
161 Tag t = new Tag((byte) 1, "myTag1");
162 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key), t.getBuffer());
163 } else {
164 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
165 }
166 }
167 return (start + n);
168 }
169
170 private void readAllRecords(HFileScanner scanner) throws IOException {
171 readAndCheckbytes(scanner, 0, 100);
172 }
173
174
175 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
176 throws IOException {
177 String value = "value";
178 int i = start;
179 for (; i < (start + n); i++) {
180 ByteBuffer key = scanner.getKey();
181 ByteBuffer val = scanner.getValue();
182 String keyStr = String.format(localFormatter, Integer.valueOf(i));
183 String valStr = value + keyStr;
184 byte [] keyBytes = Bytes.toBytes(key);
185 assertTrue("bytes for keys do not match " + keyStr + " " +
186 Bytes.toString(Bytes.toBytes(key)),
187 Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
188 byte [] valBytes = Bytes.toBytes(val);
189 assertTrue("bytes for vals do not match " + valStr + " " +
190 Bytes.toString(valBytes),
191 Arrays.equals(Bytes.toBytes(valStr), valBytes));
192 if (!scanner.next()) {
193 break;
194 }
195 }
196 assertEquals(i, start + n - 1);
197 return (start + n);
198 }
199
200 private byte[] getSomeKey(int rowId) {
201 return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
202 }
203
204 private void writeRecords(Writer writer, boolean useTags) throws IOException {
205 writeSomeRecords(writer, 0, 100, useTags);
206 writer.close();
207 }
208
209 private FSDataOutputStream createFSOutput(Path name) throws IOException {
210
211 FSDataOutputStream fout = fs.create(name);
212 return fout;
213 }
214
215
216
217
218
219 void basicWithSomeCodec(String codec, boolean useTags) throws IOException {
220 if (useTags) {
221 conf.setInt("hfile.format.version", 3);
222 }
223 if (cacheConf == null) cacheConf = new CacheConfig(conf);
224 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString() + useTags);
225 FSDataOutputStream fout = createFSOutput(ncTFile);
226 HFileContext meta = new HFileContextBuilder()
227 .withBlockSize(minBlockSize)
228 .withCompression(AbstractHFileWriter.compressionByName(codec))
229 .build();
230 Writer writer = HFile.getWriterFactory(conf, cacheConf)
231 .withOutputStream(fout)
232 .withFileContext(meta)
233
234 .withComparator(new KeyValue.RawBytesComparator())
235 .create();
236 LOG.info(writer);
237 writeRecords(writer, useTags);
238 fout.close();
239 FSDataInputStream fin = fs.open(ncTFile);
240 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
241 fs.getFileStatus(ncTFile).getLen(), cacheConf, conf);
242 System.out.println(cacheConf.toString());
243
244 reader.loadFileInfo();
245
246 HFileScanner scanner = reader.getScanner(true, false);
247
248 scanner.seekTo();
249 readAllRecords(scanner);
250 scanner.seekTo(getSomeKey(50));
251 assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
252
253 ByteBuffer readKey = scanner.getKey();
254 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
255 Bytes.toBytes(readKey)));
256
257 scanner.seekTo(new byte[0]);
258 ByteBuffer val1 = scanner.getValue();
259 scanner.seekTo(new byte[0]);
260 ByteBuffer val2 = scanner.getValue();
261 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
262
263 reader.close();
264 fin.close();
265 fs.delete(ncTFile, true);
266 }
267
268 public void testTFileFeatures() throws IOException {
269 testTFilefeaturesInternals(false);
270 testTFilefeaturesInternals(true);
271 }
272
273 protected void testTFilefeaturesInternals(boolean useTags) throws IOException {
274 basicWithSomeCodec("none", useTags);
275 basicWithSomeCodec("gz", useTags);
276 }
277
278 private void writeNumMetablocks(Writer writer, int n) {
279 for (int i = 0; i < n; i++) {
280 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
281 private int val;
282 public Writable setVal(int val) { this.val = val; return this; }
283
284 @Override
285 public void write(DataOutput out) throws IOException {
286 out.write(("something to test" + val).getBytes());
287 }
288
289 @Override
290 public void readFields(DataInput in) throws IOException { }
291 }.setVal(i));
292 }
293 }
294
295 private void someTestingWithMetaBlock(Writer writer) {
296 writeNumMetablocks(writer, 10);
297 }
298
299 private void readNumMetablocks(Reader reader, int n) throws IOException {
300 for (int i = 0; i < n; i++) {
301 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
302 ByteBuffer expected =
303 ByteBuffer.wrap(("something to test" + i).getBytes());
304 assertEquals("failed to match metadata",
305 Bytes.toStringBinary(expected), Bytes.toStringBinary(actual));
306 }
307 }
308
309 private void someReadingWithMetaBlock(Reader reader) throws IOException {
310 readNumMetablocks(reader, 10);
311 }
312
313 private void metablocks(final String compress) throws Exception {
314 if (cacheConf == null) cacheConf = new CacheConfig(conf);
315 Path mFile = new Path(ROOT_DIR, "meta.hfile");
316 FSDataOutputStream fout = createFSOutput(mFile);
317 HFileContext meta = new HFileContextBuilder()
318 .withCompression(AbstractHFileWriter.compressionByName(compress))
319 .withBlockSize(minBlockSize).build();
320 Writer writer = HFile.getWriterFactory(conf, cacheConf)
321 .withOutputStream(fout)
322 .withFileContext(meta)
323 .create();
324 someTestingWithMetaBlock(writer);
325 writer.close();
326 fout.close();
327 FSDataInputStream fin = fs.open(mFile);
328 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
329 this.fs.getFileStatus(mFile).getLen(), cacheConf, conf);
330 reader.loadFileInfo();
331
332 assertFalse(reader.getScanner(false, false).seekTo());
333 someReadingWithMetaBlock(reader);
334 fs.delete(mFile, true);
335 reader.close();
336 fin.close();
337 }
338
339
340 public void testMetaBlocks() throws Exception {
341 metablocks("none");
342 metablocks("gz");
343 }
344
345 public void testNullMetaBlocks() throws Exception {
346 if (cacheConf == null) cacheConf = new CacheConfig(conf);
347 for (Compression.Algorithm compressAlgo :
348 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
349 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
350 FSDataOutputStream fout = createFSOutput(mFile);
351 HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo)
352 .withBlockSize(minBlockSize).build();
353 Writer writer = HFile.getWriterFactory(conf, cacheConf)
354 .withOutputStream(fout)
355 .withFileContext(meta)
356 .create();
357 writer.append("foo".getBytes(), "value".getBytes());
358 writer.close();
359 fout.close();
360 Reader reader = HFile.createReader(fs, mFile, cacheConf, conf);
361 reader.loadFileInfo();
362 assertNull(reader.getMetaBlock("non-existant", false));
363 }
364 }
365
366
367
368
369 public void testCompressionOrdinance() {
370 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
371 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
372 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
373 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
374 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
375 }
376
377 }
378