1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.io.hfile;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertTrue;
21
22 import java.io.IOException;
23 import java.nio.ByteBuffer;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.io.HeapSize;
30 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
31 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
32 import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
33 import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
34 import org.apache.hadoop.hbase.testclassification.SmallTests;
35 import org.apache.hadoop.hbase.util.ChecksumType;
36 import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39 import org.junit.runner.RunWith;
40 import org.junit.runners.Parameterized;
41 import org.junit.runners.Parameterized.Parameters;
42
43 @RunWith(Parameterized.class)
44 @Category(SmallTests.class)
45 public class TestHFileDataBlockEncoder {
46 private HFileDataBlockEncoderImpl blockEncoder;
47 private RedundantKVGenerator generator = new RedundantKVGenerator();
48 private boolean includesMemstoreTS;
49
50
51
52
53
54 public TestHFileDataBlockEncoder(HFileDataBlockEncoderImpl blockEncoder,
55 boolean includesMemstoreTS) {
56 this.blockEncoder = blockEncoder;
57 this.includesMemstoreTS = includesMemstoreTS;
58 System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding()
59 + ", includesMemstoreTS: " + includesMemstoreTS);
60 }
61
62
63
64
65
66 @Test
67 public void testEncodingWithCache() throws IOException {
68 testEncodingWithCacheInternals(false);
69 testEncodingWithCacheInternals(true);
70 }
71
72 private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
73 HFileBlock block = getSampleHFileBlock(useTag);
74 HFileBlock cacheBlock = createBlockOnDisk(block, useTag);
75
76 LruBlockCache blockCache =
77 new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
78 BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
79 blockCache.cacheBlock(cacheKey, cacheBlock);
80
81 HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
82 assertTrue(heapSize instanceof HFileBlock);
83
84 HFileBlock returnedBlock = (HFileBlock) heapSize;;
85
86 if (blockEncoder.getDataBlockEncoding() ==
87 DataBlockEncoding.NONE) {
88 assertEquals(block.getBufferWithHeader(),
89 returnedBlock.getBufferWithHeader());
90 } else {
91 if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
92 System.out.println(blockEncoder);
93 }
94 assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
95 }
96 }
97
98
99 @Test
100 public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
101 testHeaderSizeInCacheWithoutChecksumInternals(false);
102 testHeaderSizeInCacheWithoutChecksumInternals(true);
103 }
104
105 private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException {
106 int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
107
108 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
109 generator.generateTestKeyValues(60, useTags), includesMemstoreTS);
110 int size = keyValues.limit();
111 ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
112 buf.position(headerSize);
113 keyValues.rewind();
114 buf.put(keyValues);
115 HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false)
116 .withIncludesMvcc(includesMemstoreTS)
117 .withIncludesTags(useTags)
118 .withBlockSize(0)
119 .withChecksumType(ChecksumType.NULL)
120 .build();
121 HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
122 HFileBlock.FILL_HEADER, 0,
123 0, hfileContext);
124 HFileBlock cacheBlock = createBlockOnDisk(block, useTags);
125 assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
126 }
127
128 private HFileBlock createBlockOnDisk(HFileBlock block, boolean useTags) throws IOException {
129 int size;
130 HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
131 blockEncoder.getDataBlockEncoding(),
132 HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext());
133 context.setDummyHeader(block.getDummyHeaderForVersion());
134 blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), context, block.getBlockType());
135 byte[] encodedBytes = context.getUncompressedBytesWithHeader();
136 size = encodedBytes.length - block.getDummyHeaderForVersion().length;
137 return new HFileBlock(context.getBlockType(), size, size, -1,
138 ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0,
139 block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
140 }
141
142
143
144
145
146 @Test
147 public void testEncoding() throws IOException {
148 testEncodingInternals(false);
149 testEncodingInternals(true);
150 }
151
152 private void testEncodingInternals(boolean useTag) throws IOException {
153
154 HFileBlock block = getSampleHFileBlock(useTag);
155 HFileBlock blockOnDisk = createBlockOnDisk(block, useTag);
156
157 if (blockEncoder.getDataBlockEncoding() !=
158 DataBlockEncoding.NONE) {
159 assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
160 assertEquals(blockEncoder.getDataBlockEncoding().getId(),
161 blockOnDisk.getDataBlockEncodingId());
162 } else {
163 assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
164 }
165 }
166
167 private HFileBlock getSampleHFileBlock(boolean useTag) {
168 ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
169 generator.generateTestKeyValues(60, useTag), includesMemstoreTS);
170 int size = keyValues.limit();
171 ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
172 buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
173 keyValues.rewind();
174 buf.put(keyValues);
175 HFileContext meta = new HFileContextBuilder()
176 .withIncludesMvcc(includesMemstoreTS)
177 .withIncludesTags(useTag)
178 .withHBaseCheckSum(true)
179 .withCompression(Algorithm.NONE)
180 .withBlockSize(0)
181 .withChecksumType(ChecksumType.NULL)
182 .build();
183 HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
184 HFileBlock.FILL_HEADER, 0,
185 0, meta);
186 return b;
187 }
188
189
190
191
192 @Parameters
193 public static Collection<Object[]> getAllConfigurations() {
194 List<Object[]> configurations =
195 new ArrayList<Object[]>();
196
197 for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
198 for (boolean includesMemstoreTS : new boolean[] {false, true}) {
199 configurations.add(new Object[] {
200 new HFileDataBlockEncoderImpl(diskAlgo),
201 new Boolean(includesMemstoreTS)});
202 }
203 }
204
205 return configurations;
206 }
207 }