View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.io;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
36  import org.apache.hadoop.hbase.io.hfile.HFile;
37  import org.apache.hadoop.hbase.io.hfile.HFileContext;
38  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
39  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
40  import org.apache.hadoop.hbase.testclassification.SmallTests;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.junit.AfterClass;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  @Category(SmallTests.class)
48  public class TestHalfStoreFileReader {
49    private static HBaseTestingUtility TEST_UTIL;
50  
51    @BeforeClass
52    public static void setupBeforeClass() throws Exception {
53      TEST_UTIL = new HBaseTestingUtility();
54    }
55  
56    @AfterClass
57    public static void tearDownAfterClass() throws Exception {
58      TEST_UTIL.cleanupTestDir();
59    }
60  
61    /**
62     * Test the scanner and reseek of a half hfile scanner. The scanner API
63     * demands that seekTo and reseekTo() only return < 0 if the key lies
64     * before the start of the file (with no position on the scanner). Returning
65     * 0 if perfect match (rare), and return > 1 if we got an imperfect match.
66     *
67     * The latter case being the most common, we should generally be returning 1,
68     * and if we do, there may or may not be a 'next' in the scanner/file.
69     *
70     * A bug in the half file scanner was returning -1 at the end of the bottom
71     * half, and that was causing the infrastructure above to go null causing NPEs
72     * and other problems.  This test reproduces that failure, and also tests
73     * both the bottom and top of the file while we are at it.
74     *
75     * @throws IOException
76     */
77    @Test
78    public void testHalfScanAndReseek() throws IOException {
79      String root_dir = TEST_UTIL.getDataTestDir().toString();
80      Path p = new Path(root_dir, "test");
81  
82      Configuration conf = TEST_UTIL.getConfiguration();
83      FileSystem fs = FileSystem.get(conf);
84      CacheConfig cacheConf = new CacheConfig(conf);
85      HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
86      HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
87          .withPath(fs, p)
88          .withFileContext(meta)
89          .create();
90  
91      // write some things.
92      List<KeyValue> items = genSomeKeys();
93      for (KeyValue kv : items) {
94        w.append(kv);
95      }
96      w.close();
97  
98      HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
99      r.loadFileInfo();
100     byte [] midkey = r.midkey();
101     KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
102     midkey = midKV.getRow();
103 
104     //System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
105 
106     Reference bottom = new Reference(midkey, Reference.Range.bottom);
107     doTestOfScanAndReseek(p, fs, bottom, cacheConf);
108 
109     Reference top = new Reference(midkey, Reference.Range.top);
110     doTestOfScanAndReseek(p, fs, top, cacheConf);
111 
112     r.close();
113   }
114 
115   private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom,
116       CacheConfig cacheConf)
117       throws IOException {
118     final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
119       cacheConf, bottom, TEST_UTIL.getConfiguration());
120     halfreader.loadFileInfo();
121     final HFileScanner scanner = halfreader.getScanner(false, false);
122 
123     scanner.seekTo();
124     KeyValue curr;
125     do {
126       curr = scanner.getKeyValue();
127       KeyValue reseekKv =
128           getLastOnCol(curr);
129       int ret = scanner.reseekTo(reseekKv.getKey());
130       assertTrue("reseek to returned: " + ret, ret > 0);
131       //System.out.println(curr + ": " + ret);
132     } while (scanner.next());
133 
134     int ret = scanner.reseekTo(getLastOnCol(curr).getKey());
135     //System.out.println("Last reseek: " + ret);
136     assertTrue( ret > 0 );
137 
138     halfreader.close(true);
139   }
140 
141 
142   // Tests the scanner on an HFile that is backed by HalfStoreFiles
143   @Test
144   public void testHalfScanner() throws IOException {
145       String root_dir = TEST_UTIL.getDataTestDir().toString();
146       Path p = new Path(root_dir, "test");
147       Configuration conf = TEST_UTIL.getConfiguration();
148       FileSystem fs = FileSystem.get(conf);
149       CacheConfig cacheConf = new CacheConfig(conf);
150       HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build();
151       HFile.Writer w = HFile.getWriterFactory(conf, cacheConf)
152               .withPath(fs, p)
153               .withFileContext(meta)
154               .create();
155 
156       // write some things.
157       List<KeyValue> items = genSomeKeys();
158       for (KeyValue kv : items) {
159           w.append(kv);
160       }
161       w.close();
162 
163 
164       HFile.Reader r = HFile.createReader(fs, p, cacheConf, conf);
165       r.loadFileInfo();
166       byte[] midkey = r.midkey();
167       KeyValue midKV = KeyValue.createKeyValueFromKey(midkey);
168       midkey = midKV.getRow();
169 
170       Reference bottom = new Reference(midkey, Reference.Range.bottom);
171       Reference top = new Reference(midkey, Reference.Range.top);
172 
173       // Ugly code to get the item before the midkey
174       KeyValue beforeMidKey = null;
175       for (KeyValue item : items) {
176           if (KeyValue.COMPARATOR.compare(item, midKV) >= 0) {
177               break;
178           }
179           beforeMidKey = item;
180       }
181       System.out.println("midkey: " + midKV + " or: " + Bytes.toStringBinary(midkey));
182       System.out.println("beforeMidKey: " + beforeMidKey);
183 
184 
185       // Seek on the splitKey, should be in top, not in bottom
186       KeyValue foundKeyValue = doTestOfSeekBefore(p, fs, bottom, midKV, cacheConf);
187       assertEquals(beforeMidKey, foundKeyValue);
188 
189       // Seek tot the last thing should be the penultimate on the top, the one before the midkey on the bottom.
190       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(items.size() - 1), cacheConf);
191       assertEquals(items.get(items.size() - 2), foundKeyValue);
192 
193       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(items.size() - 1), cacheConf);
194       assertEquals(beforeMidKey, foundKeyValue);
195 
196       // Try and seek before something that is in the bottom.
197       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(0), cacheConf);
198       assertNull(foundKeyValue);
199 
200       // Try and seek before the first thing.
201       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(0), cacheConf);
202       assertNull(foundKeyValue);
203 
204       // Try and seek before the second thing in the top and bottom.
205       foundKeyValue = doTestOfSeekBefore(p, fs, top, items.get(1), cacheConf);
206       assertNull(foundKeyValue);
207 
208       foundKeyValue = doTestOfSeekBefore(p, fs, bottom, items.get(1), cacheConf);
209       assertEquals(items.get(0), foundKeyValue);
210 
211       // Try to seek before the splitKey in the top file
212       foundKeyValue = doTestOfSeekBefore(p, fs, top, midKV, cacheConf);
213       assertNull(foundKeyValue);
214     }
215 
216   private KeyValue doTestOfSeekBefore(Path p, FileSystem fs, Reference bottom, KeyValue seekBefore,
217                                         CacheConfig cacheConfig)
218             throws IOException {
219       final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p,
220               cacheConfig, bottom, TEST_UTIL.getConfiguration());
221       halfreader.loadFileInfo();
222       final HFileScanner scanner = halfreader.getScanner(false, false);
223       scanner.seekBefore(seekBefore.getKey());
224       return scanner.getKeyValue();
225   }
226 
227   private KeyValue getLastOnCol(KeyValue curr) {
228     return KeyValue.createLastOnRow(
229         curr.getBuffer(), curr.getRowOffset(), curr.getRowLength(),
230         curr.getBuffer(), curr.getFamilyOffset(), curr.getFamilyLength(),
231         curr.getBuffer(), curr.getQualifierOffset(), curr.getQualifierLength());
232   }
233 
234   static final int SIZE = 1000;
235 
236   static byte[] _b(String s) {
237     return Bytes.toBytes(s);
238   }
239 
240   List<KeyValue> genSomeKeys() {
241     List<KeyValue> ret = new ArrayList<KeyValue>(SIZE);
242     for (int i = 0; i < SIZE; i++) {
243       KeyValue kv =
244           new KeyValue(
245               _b(String.format("row_%04d", i)),
246               _b("family"),
247               _b("qualifier"),
248               1000, // timestamp
249               _b("value"));
250       ret.add(kv);
251     }
252     return ret;
253   }
254 
255 
256 
257 }
258