View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.regionserver;
18  
19  import java.io.IOException;
20  import java.util.ArrayList;
21  import java.util.List;
22  import java.util.Random;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.Path;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HConstants;
28  import org.apache.hadoop.hbase.KeyValue;
29  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
30  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
31  import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
32  
33  /**
34   * Test seek performance for encoded data blocks. Read an HFile and do several
35   * random seeks.
36   */
37  public class EncodedSeekPerformanceTest {
38    private static final double NANOSEC_IN_SEC = 1000.0 * 1000.0 * 1000.0;
39    private static final double BYTES_IN_MEGABYTES = 1024.0 * 1024.0;
40    /** Default number of seeks which will be used in benchmark. */
41    public static int DEFAULT_NUMBER_OF_SEEKS = 10000;
42  
43    private final HBaseTestingUtility testingUtility = new HBaseTestingUtility();
44    private Configuration configuration = testingUtility.getConfiguration();
45    private CacheConfig cacheConf = new CacheConfig(configuration);
46    private Random randomizer;
47    private int numberOfSeeks;
48  
49    /** Use this benchmark with default options */
50    public EncodedSeekPerformanceTest() {
51      configuration.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.5f);
52      randomizer = new Random(42l);
53      numberOfSeeks = DEFAULT_NUMBER_OF_SEEKS;
54    }
55  
56    private List<KeyValue> prepareListOfTestSeeks(Path path) throws IOException {
57      List<KeyValue> allKeyValues = new ArrayList<KeyValue>();
58  
59      // read all of the key values
60      StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
61          path, configuration, cacheConf, BloomType.NONE);
62  
63      StoreFile.Reader reader = storeFile.createReader();
64      StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
65      KeyValue current;
66  
67      scanner.seek(KeyValue.LOWESTKEY);
68      while (null != (current = scanner.next())) {
69        allKeyValues.add(current);
70      }
71  
72      storeFile.closeReader(cacheConf.shouldEvictOnClose());
73  
74      // pick seeks by random
75      List<KeyValue> seeks = new ArrayList<KeyValue>();
76      for (int i = 0; i < numberOfSeeks; ++i) {
77        KeyValue keyValue = allKeyValues.get(
78            randomizer.nextInt(allKeyValues.size()));
79        seeks.add(keyValue);
80      }
81  
82      clearBlockCache();
83  
84      return seeks;
85    }
86  
87    private void runTest(Path path, DataBlockEncoding blockEncoding,
88        List<KeyValue> seeks) throws IOException {
89      // read all of the key values
90      StoreFile storeFile = new StoreFile(testingUtility.getTestFileSystem(),
91        path, configuration, cacheConf, BloomType.NONE);
92  
93      long totalSize = 0;
94  
95      StoreFile.Reader reader = storeFile.createReader();
96      StoreFileScanner scanner = reader.getStoreFileScanner(true, false);
97  
98      long startReadingTime = System.nanoTime();
99      KeyValue current;
100     scanner.seek(KeyValue.LOWESTKEY);
101     while (null != (current = scanner.next())) { // just iterate it!
102       if (current.getLength() < 0) {
103         throw new IOException("Negative KV size: " + current);
104       }
105       totalSize += current.getLength();
106     }
107     long finishReadingTime = System.nanoTime();
108 
109     // do seeks
110     long startSeeksTime = System.nanoTime();
111     for (KeyValue keyValue : seeks) {
112       scanner.seek(keyValue);
113       KeyValue toVerify = scanner.next();
114       if (!keyValue.equals(toVerify)) {
115         System.out.println(String.format("KeyValue doesn't match:\n" +
116             "Orig key: %s\n" +
117             "Ret key:  %s", keyValue.getKeyString(), toVerify.getKeyString()));
118         break;
119       }
120     }
121     long finishSeeksTime = System.nanoTime();
122     if (finishSeeksTime < startSeeksTime) {
123       throw new AssertionError("Finish time " + finishSeeksTime +
124           " is earlier than start time " + startSeeksTime);
125     }
126 
127     // write some stats
128     double readInMbPerSec = (totalSize * NANOSEC_IN_SEC) /
129         (BYTES_IN_MEGABYTES * (finishReadingTime - startReadingTime));
130     double seeksPerSec = (seeks.size() * NANOSEC_IN_SEC) /
131         (finishSeeksTime - startSeeksTime);
132 
133     storeFile.closeReader(cacheConf.shouldEvictOnClose());
134     clearBlockCache();
135 
136     System.out.println(blockEncoding);
137     System.out.printf("  Read speed:       %8.2f (MB/s)\n", readInMbPerSec);
138     System.out.printf("  Seeks per second: %8.2f (#/s)\n", seeksPerSec);
139     System.out.printf("  Total KV size:    %d\n", totalSize);
140   }
141 
142   /**
143    * @param path Path to the HFile which will be used.
144    * @param encoders List of encoders which will be used for tests.
145    * @throws IOException if there is a bug while reading from disk
146    */
147   public void runTests(Path path, DataBlockEncoding[] encodings)
148       throws IOException {
149     List<KeyValue> seeks = prepareListOfTestSeeks(path);
150 
151     for (DataBlockEncoding blockEncoding : encodings) {
152       runTest(path, blockEncoding, seeks);
153     }
154   }
155 
156   /**
157    * Command line interface:
158    * @param args Takes one argument - file size.
159    * @throws IOException if there is a bug while reading from disk
160    */
161   public static void main(final String[] args) throws IOException {
162     if (args.length < 1) {
163       printUsage();
164       System.exit(-1);
165     }
166 
167     Path path = new Path(args[0]);
168 
169     // TODO, this test doesn't work as expected any more. Need to fix.
170     EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest();
171     utility.runTests(path, DataBlockEncoding.values());
172 
173     System.exit(0);
174   }
175 
176   private static void printUsage() {
177     System.out.println("Usage: one argument, name of the HFile");
178   }
179 
180   private void clearBlockCache() {
181     ((LruBlockCache) cacheConf.getBlockCache()).clearCache();
182   }
183 }