View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.io.hfile;
18  
19  import static org.junit.Assert.assertEquals;
20  import static org.junit.Assert.assertTrue;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.Collection;
26  import java.util.List;
27  
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.io.HeapSize;
30  import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
31  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
32  import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
33  import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
34  import org.apache.hadoop.hbase.testclassification.SmallTests;
35  import org.apache.hadoop.hbase.util.ChecksumType;
36  import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  import org.junit.runner.RunWith;
40  import org.junit.runners.Parameterized;
41  import org.junit.runners.Parameterized.Parameters;
42  
43  @RunWith(Parameterized.class)
44  @Category(SmallTests.class)
45  public class TestHFileDataBlockEncoder {
46    private HFileDataBlockEncoderImpl blockEncoder;
47    private RedundantKVGenerator generator = new RedundantKVGenerator();
48    private boolean includesMemstoreTS;
49  
50    /**
51     * Create test for given data block encoding configuration.
52     * @param blockEncoder What kind of encoding policy will be used.
53     */
54    public TestHFileDataBlockEncoder(HFileDataBlockEncoderImpl blockEncoder,
55        boolean includesMemstoreTS) {
56      this.blockEncoder = blockEncoder;
57      this.includesMemstoreTS = includesMemstoreTS;
58      System.err.println("Encoding: " + blockEncoder.getDataBlockEncoding()
59          + ", includesMemstoreTS: " + includesMemstoreTS);
60    }
61  
62    /**
63     * Test putting and taking out blocks into cache with different
64     * encoding options.
65     */
66    @Test
67    public void testEncodingWithCache() throws IOException {
68      testEncodingWithCacheInternals(false);
69      testEncodingWithCacheInternals(true);
70    }
71  
72    private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
73      HFileBlock block = getSampleHFileBlock(useTag);
74      HFileBlock cacheBlock = createBlockOnDisk(block, useTag);
75  
76      LruBlockCache blockCache =
77          new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
78      BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
79      blockCache.cacheBlock(cacheKey, cacheBlock);
80  
81      HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
82      assertTrue(heapSize instanceof HFileBlock);
83  
84      HFileBlock returnedBlock = (HFileBlock) heapSize;;
85  
86      if (blockEncoder.getDataBlockEncoding() ==
87          DataBlockEncoding.NONE) {
88        assertEquals(block.getBufferWithHeader(),
89            returnedBlock.getBufferWithHeader());
90      } else {
91        if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
92          System.out.println(blockEncoder);
93        }
94        assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
95      }
96    }
97  
98    /** Test for HBASE-5746. */
99    @Test
100   public void testHeaderSizeInCacheWithoutChecksum() throws Exception {
101     testHeaderSizeInCacheWithoutChecksumInternals(false);
102     testHeaderSizeInCacheWithoutChecksumInternals(true);
103   }
104 
105   private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException {
106     int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
107     // Create some KVs and create the block with old-style header.
108     ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
109         generator.generateTestKeyValues(60, useTags), includesMemstoreTS);
110     int size = keyValues.limit();
111     ByteBuffer buf = ByteBuffer.allocate(size + headerSize);
112     buf.position(headerSize);
113     keyValues.rewind();
114     buf.put(keyValues);
115     HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false)
116                         .withIncludesMvcc(includesMemstoreTS)
117                         .withIncludesTags(useTags)
118                         .withBlockSize(0)
119                         .withChecksumType(ChecksumType.NULL)
120                         .build();
121     HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
122         HFileBlock.FILL_HEADER, 0,
123         0, hfileContext);
124     HFileBlock cacheBlock = createBlockOnDisk(block, useTags);
125     assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length);
126   }
127 
128   private HFileBlock createBlockOnDisk(HFileBlock block, boolean useTags) throws IOException {
129     int size;
130     HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
131         blockEncoder.getDataBlockEncoding(),
132         HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext());
133     context.setDummyHeader(block.getDummyHeaderForVersion());
134     blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(), context, block.getBlockType());
135     byte[] encodedBytes = context.getUncompressedBytesWithHeader();
136     size = encodedBytes.length - block.getDummyHeaderForVersion().length;
137     return new HFileBlock(context.getBlockType(), size, size, -1,
138             ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0,
139             block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
140   }
141 
142   /**
143    * Test encoding.
144    * @throws IOException
145    */
146   @Test
147   public void testEncoding() throws IOException {
148     testEncodingInternals(false);
149     testEncodingInternals(true);
150   }
151 
152   private void testEncodingInternals(boolean useTag) throws IOException {
153     // usually we have just block without headers, but don't complicate that
154     HFileBlock block = getSampleHFileBlock(useTag);
155     HFileBlock blockOnDisk = createBlockOnDisk(block, useTag);
156 
157     if (blockEncoder.getDataBlockEncoding() !=
158         DataBlockEncoding.NONE) {
159       assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
160       assertEquals(blockEncoder.getDataBlockEncoding().getId(),
161           blockOnDisk.getDataBlockEncodingId());
162     } else {
163       assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
164     }
165   }
166 
167   private HFileBlock getSampleHFileBlock(boolean useTag) {
168     ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(
169         generator.generateTestKeyValues(60, useTag), includesMemstoreTS);
170     int size = keyValues.limit();
171     ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE);
172     buf.position(HConstants.HFILEBLOCK_HEADER_SIZE);
173     keyValues.rewind();
174     buf.put(keyValues);
175     HFileContext meta = new HFileContextBuilder()
176                         .withIncludesMvcc(includesMemstoreTS)
177                         .withIncludesTags(useTag)
178                         .withHBaseCheckSum(true)
179                         .withCompression(Algorithm.NONE)
180                         .withBlockSize(0)
181                         .withChecksumType(ChecksumType.NULL)
182                         .build();
183     HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
184         HFileBlock.FILL_HEADER, 0, 
185          0, meta);
186     return b;
187   }
188 
189   /**
190    * @return All possible data block encoding configurations
191    */
192   @Parameters
193   public static Collection<Object[]> getAllConfigurations() {
194     List<Object[]> configurations =
195         new ArrayList<Object[]>();
196 
197     for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
198       for (boolean includesMemstoreTS : new boolean[] {false, true}) {
199         configurations.add(new Object[] {
200             new HFileDataBlockEncoderImpl(diskAlgo),
201             new Boolean(includesMemstoreTS)});
202       }
203     }
204 
205     return configurations;
206   }
207 }