View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.lang.management.ManagementFactory;
22  import java.lang.management.MemoryUsage;
23  
24  import com.google.common.annotations.VisibleForTesting;
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
32  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
33  import org.apache.hadoop.hbase.util.DirectMemoryUtils;
34  import org.apache.hadoop.util.StringUtils;
35  
36  /**
37   * Stores all of the cache objects and configuration for a single HFile.
38   */
39  @InterfaceAudience.Private
40  public class CacheConfig {
41    private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
42  
43    /**
44     * Configuration key to cache data blocks on write. There are separate
45     * switches for bloom blocks and non-root index blocks.
46     */
47    public static final String CACHE_BLOCKS_ON_WRITE_KEY =
48        "hbase.rs.cacheblocksonwrite";
49  
50    /**
51     * Configuration key to cache leaf and intermediate-level index blocks on
52     * write.
53     */
54    public static final String CACHE_INDEX_BLOCKS_ON_WRITE_KEY =
55        "hfile.block.index.cacheonwrite";
56  
57    /**
58     * Configuration key to cache compound bloom filter blocks on write.
59     */
60    public static final String CACHE_BLOOM_BLOCKS_ON_WRITE_KEY =
61        "hfile.block.bloom.cacheonwrite";
62  
63    /**
64     * Configuration key to cache data blocks in compressed and/or encrypted format.
65     */
66    public static final String CACHE_DATA_BLOCKS_COMPRESSED_KEY =
67        "hbase.block.data.cachecompressed";
68  
69    /**
70     * Configuration key to evict all blocks of a given file from the block cache
71     * when the file is closed.
72     */
73    public static final String EVICT_BLOCKS_ON_CLOSE_KEY =
74        "hbase.rs.evictblocksonclose";
75  
76    /**
77     * Configuration keys for Bucket cache
78     */
79    public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
80    public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
81    public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = 
82        "hbase.bucketcache.persistent.path";
83    public static final String BUCKET_CACHE_COMBINED_KEY = 
84        "hbase.bucketcache.combinedcache.enabled";
85    public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = 
86        "hbase.bucketcache.percentage.in.combinedcache";
87    public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
88    public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = 
89        "hbase.bucketcache.writer.queuelength";
90  
91    /**
92     * A comma-delimited array of values for use as bucket sizes.
93     */
94    public static final String BUCKET_CACHE_BUCKETS_KEY = "hbase.bucketcache.bucket.sizes";
95  
96    /**
97     * Defaults for Bucket cache
98     */
99    public static final boolean DEFAULT_BUCKET_CACHE_COMBINED = true;
100   public static final int DEFAULT_BUCKET_CACHE_WRITER_THREADS = 3;
101   public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
102   public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
103 
104  /**
105    * Configuration key to prefetch all blocks of a given file into the block cache
106    * when the file is opened.
107    */
108   public static final String PREFETCH_BLOCKS_ON_OPEN_KEY =
109       "hbase.rs.prefetchblocksonopen";
110 
111   /**
112    * The target block size used by blockcache instances. Defaults to
113    * {@link HConstants#DEFAULT_BLOCKSIZE}.
114    * TODO: this config point is completely wrong, as it's used to determine the
115    * target block size of BlockCache instances. Rename.
116    */
117   public static final String BLOCKCACHE_BLOCKSIZE_KEY = "hbase.offheapcache.minblocksize";
118 
119   // Defaults
120 
121   public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
122   public static final boolean DEFAULT_CACHE_DATA_ON_WRITE = false;
123   public static final boolean DEFAULT_IN_MEMORY = false;
124   public static final boolean DEFAULT_CACHE_INDEXES_ON_WRITE = false;
125   public static final boolean DEFAULT_CACHE_BLOOMS_ON_WRITE = false;
126   public static final boolean DEFAULT_EVICT_ON_CLOSE = false;
127   public static final boolean DEFAULT_CACHE_DATA_COMPRESSED = false;
128   public static final boolean DEFAULT_PREFETCH_ON_OPEN = false;
129 
130   /** Local reference to the block cache, null if completely disabled */
131   private final BlockCache blockCache;
132 
133   /**
134    * Whether blocks should be cached on read (default is on if there is a
135    * cache but this can be turned off on a per-family or per-request basis)
136    */
137   private boolean cacheDataOnRead;
138 
139   /** Whether blocks should be flagged as in-memory when being cached */
140   private final boolean inMemory;
141 
142   /** Whether data blocks should be cached when new files are written */
143   private boolean cacheDataOnWrite;
144 
145   /** Whether index blocks should be cached when new files are written */
146   private final boolean cacheIndexesOnWrite;
147 
148   /** Whether compound bloom filter blocks should be cached on write */
149   private final boolean cacheBloomsOnWrite;
150 
151   /** Whether blocks of a file should be evicted when the file is closed */
152   private boolean evictOnClose;
153 
154   /** Whether data blocks should be stored in compressed and/or encrypted form in the cache */
155   private final boolean cacheDataCompressed;
156 
157   /** Whether data blocks should be prefetched into the cache */
158   private final boolean prefetchOnOpen;
159 
160   /**
161    * Create a cache configuration using the specified configuration object and
162    * family descriptor.
163    * @param conf hbase configuration
164    * @param family column family configuration
165    */
166   public CacheConfig(Configuration conf, HColumnDescriptor family) {
167     this(CacheConfig.instantiateBlockCache(conf),
168         family.isBlockCacheEnabled(),
169         family.isInMemory(),
170         // For the following flags we enable them regardless of per-schema settings
171         // if they are enabled in the global configuration.
172         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY,
173             DEFAULT_CACHE_DATA_ON_WRITE) || family.shouldCacheDataOnWrite(),
174         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
175             DEFAULT_CACHE_INDEXES_ON_WRITE) || family.shouldCacheIndexesOnWrite(),
176         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
177             DEFAULT_CACHE_BLOOMS_ON_WRITE) || family.shouldCacheBloomsOnWrite(),
178         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY,
179             DEFAULT_EVICT_ON_CLOSE) || family.shouldEvictBlocksOnClose(),
180         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
181         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY,
182             DEFAULT_PREFETCH_ON_OPEN) || family.shouldPrefetchBlocksOnOpen()
183      );
184   }
185 
186   /**
187    * Create a cache configuration using the specified configuration object and
188    * defaults for family level settings.
189    * @param conf hbase configuration
190    */
191   public CacheConfig(Configuration conf) {
192     this(CacheConfig.instantiateBlockCache(conf),
193         DEFAULT_CACHE_DATA_ON_READ,
194         DEFAULT_IN_MEMORY, // This is a family-level setting so can't be set
195                            // strictly from conf
196         conf.getBoolean(CACHE_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_DATA_ON_WRITE),
197         conf.getBoolean(CACHE_INDEX_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_INDEXES_ON_WRITE),
198         conf.getBoolean(CACHE_BLOOM_BLOCKS_ON_WRITE_KEY, DEFAULT_CACHE_BLOOMS_ON_WRITE),
199         conf.getBoolean(EVICT_BLOCKS_ON_CLOSE_KEY, DEFAULT_EVICT_ON_CLOSE),
200         conf.getBoolean(CACHE_DATA_BLOCKS_COMPRESSED_KEY, DEFAULT_CACHE_DATA_COMPRESSED),
201         conf.getBoolean(PREFETCH_BLOCKS_ON_OPEN_KEY, DEFAULT_PREFETCH_ON_OPEN)
202     );
203   }
204 
205   /**
206    * Create a block cache configuration with the specified cache and
207    * configuration parameters.
208    * @param blockCache reference to block cache, null if completely disabled
209    * @param cacheDataOnRead whether data blocks should be cached on read
210    * @param inMemory whether blocks should be flagged as in-memory
211    * @param cacheDataOnWrite whether data blocks should be cached on write
212    * @param cacheIndexesOnWrite whether index blocks should be cached on write
213    * @param cacheBloomsOnWrite whether blooms should be cached on write
214    * @param evictOnClose whether blocks should be evicted when HFile is closed
215    * @param cacheDataCompressed whether to store blocks as compressed in the cache
216    * @param prefetchOnOpen whether to prefetch blocks upon open
217    */
218   CacheConfig(final BlockCache blockCache,
219       final boolean cacheDataOnRead, final boolean inMemory,
220       final boolean cacheDataOnWrite, final boolean cacheIndexesOnWrite,
221       final boolean cacheBloomsOnWrite, final boolean evictOnClose,
222       final boolean cacheDataCompressed, final boolean prefetchOnOpen) {
223     this.blockCache = blockCache;
224     this.cacheDataOnRead = cacheDataOnRead;
225     this.inMemory = inMemory;
226     this.cacheDataOnWrite = cacheDataOnWrite;
227     this.cacheIndexesOnWrite = cacheIndexesOnWrite;
228     this.cacheBloomsOnWrite = cacheBloomsOnWrite;
229     this.evictOnClose = evictOnClose;
230     this.cacheDataCompressed = cacheDataCompressed;
231     this.prefetchOnOpen = prefetchOnOpen;
232   }
233 
234   /**
235    * Constructs a cache configuration copied from the specified configuration.
236    * @param cacheConf
237    */
238   public CacheConfig(CacheConfig cacheConf) {
239     this(cacheConf.blockCache, cacheConf.cacheDataOnRead, cacheConf.inMemory,
240         cacheConf.cacheDataOnWrite, cacheConf.cacheIndexesOnWrite,
241         cacheConf.cacheBloomsOnWrite, cacheConf.evictOnClose,
242         cacheConf.cacheDataCompressed, cacheConf.prefetchOnOpen);
243   }
244 
245   /**
246    * Checks whether the block cache is enabled.
247    */
248   public boolean isBlockCacheEnabled() {
249     return this.blockCache != null;
250   }
251 
252   /**
253    * Returns the block cache.
254    * @return the block cache, or null if caching is completely disabled
255    */
256   public BlockCache getBlockCache() {
257     return this.blockCache;
258   }
259 
260   /**
261    * Returns whether the blocks of this HFile should be cached on read or not.
262    * @return true if blocks should be cached on read, false if not
263    */
264   public boolean shouldCacheDataOnRead() {
265     return isBlockCacheEnabled() && cacheDataOnRead;
266   }
267 
268   /**
269    * Should we cache a block of a particular category? We always cache
270    * important blocks such as index blocks, as long as the block cache is
271    * available.
272    */
273   public boolean shouldCacheBlockOnRead(BlockCategory category) {
274     return isBlockCacheEnabled()
275         && (cacheDataOnRead ||
276             category == BlockCategory.INDEX ||
277             category == BlockCategory.BLOOM ||
278             (prefetchOnOpen &&
279                 (category != BlockCategory.META &&
280                  category != BlockCategory.UNKNOWN)));
281   }
282 
283   /**
284    * @return true if blocks in this file should be flagged as in-memory
285    */
286   public boolean isInMemory() {
287     return isBlockCacheEnabled() && this.inMemory;
288   }
289 
290   /**
291    * @return true if data blocks should be written to the cache when an HFile is
292    *         written, false if not
293    */
294   public boolean shouldCacheDataOnWrite() {
295     return isBlockCacheEnabled() && this.cacheDataOnWrite;
296   }
297 
298   /**
299    * Only used for testing.
300    * @param cacheDataOnWrite whether data blocks should be written to the cache
301    *                         when an HFile is written
302    */
303   public void setCacheDataOnWrite(boolean cacheDataOnWrite) {
304     this.cacheDataOnWrite = cacheDataOnWrite;
305   }
306 
307   /**
308    * @return true if index blocks should be written to the cache when an HFile
309    *         is written, false if not
310    */
311   public boolean shouldCacheIndexesOnWrite() {
312     return isBlockCacheEnabled() && this.cacheIndexesOnWrite;
313   }
314 
315   /**
316    * @return true if bloom blocks should be written to the cache when an HFile
317    *         is written, false if not
318    */
319   public boolean shouldCacheBloomsOnWrite() {
320     return isBlockCacheEnabled() && this.cacheBloomsOnWrite;
321   }
322 
323   /**
324    * @return true if blocks should be evicted from the cache when an HFile
325    *         reader is closed, false if not
326    */
327   public boolean shouldEvictOnClose() {
328     return isBlockCacheEnabled() && this.evictOnClose;
329   }
330 
331   /**
332    * Only used for testing.
333    * @param evictOnClose whether blocks should be evicted from the cache when an
334    *                     HFile reader is closed
335    */
336   public void setEvictOnClose(boolean evictOnClose) {
337     this.evictOnClose = evictOnClose;
338   }
339 
340   /**
341    * @return true if data blocks should be compressed in the cache, false if not
342    */
343   public boolean shouldCacheDataCompressed() {
344     return isBlockCacheEnabled() && this.cacheDataCompressed;
345   }
346 
347   /**
348    * @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
349    */
350   public boolean shouldCacheCompressed(BlockCategory category) {
351     if (!isBlockCacheEnabled()) return false;
352     switch (category) {
353       case DATA:
354         return this.cacheDataCompressed;
355       default:
356         return false;
357     }
358   }
359 
360   /**
361    * @return true if blocks should be prefetched into the cache on open, false if not
362    */
363   public boolean shouldPrefetchOnOpen() {
364     return isBlockCacheEnabled() && this.prefetchOnOpen;
365   }
366 
367   @Override
368   public String toString() {
369     if (!isBlockCacheEnabled()) {
370       return "CacheConfig:disabled";
371     }
372     return "CacheConfig:enabled " +
373       "[cacheDataOnRead=" + shouldCacheDataOnRead() + "] " +
374       "[cacheDataOnWrite=" + shouldCacheDataOnWrite() + "] " +
375       "[cacheIndexesOnWrite=" + shouldCacheIndexesOnWrite() + "] " +
376       "[cacheBloomsOnWrite=" + shouldCacheBloomsOnWrite() + "] " +
377       "[cacheEvictOnClose=" + shouldEvictOnClose() + "] " +
378       "[cacheDataCompressed=" + shouldCacheDataCompressed() + "] " +
379       "[prefetchOnOpen=" + shouldPrefetchOnOpen() + "]";
380   }
381 
382   // Static block cache reference and methods
383 
384   /**
385    * Static reference to the block cache, or null if no caching should be used
386    * at all.
387    */
388   @VisibleForTesting
389   static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
390 
391   /** Boolean whether we have disabled the block cache entirely. */
392   private static boolean blockCacheDisabled = false;
393 
394   /**
395    * Returns the block cache or <code>null</code> in case none should be used.
396    *
397    * @param conf  The current configuration.
398    * @return The block cache or <code>null</code>.
399    */
400   private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
401     if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
402     if (blockCacheDisabled) return null;
403 
404     float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
405       HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
406     if (cachePercentage == 0L) {
407       blockCacheDisabled = true;
408       return null;
409     }
410     if (cachePercentage > 1.0) {
411       throw new IllegalArgumentException(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY +
412         " must be between 0.0 and 1.0, and not > 1.0");
413     }
414 
415     // Calculate the amount of heap to give the heap.
416     MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
417     long lruCacheSize = (long) (mu.getMax() * cachePercentage);
418     int blockSize = conf.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
419     long offHeapCacheSize =
420       (long) (conf.getFloat("hbase.offheapcache.percentage", (float) 0) *
421           DirectMemoryUtils.getDirectMemorySize());
422     if (offHeapCacheSize <= 0) {
423       String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null);
424       float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
425       // A percentage of max heap size or a absolute value with unit megabytes
426       long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax()
427           * bucketCachePercentage : bucketCachePercentage * 1024 * 1024);
428 
429       boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
430           DEFAULT_BUCKET_CACHE_COMBINED);
431       BucketCache bucketCache = null;
432       if (bucketCacheIOEngineName != null && bucketCacheSize > 0) {
433         int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY,
434             DEFAULT_BUCKET_CACHE_WRITER_THREADS);
435         int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY,
436             DEFAULT_BUCKET_CACHE_WRITER_QUEUE);
437         String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY);
438         float combinedPercentage = conf.getFloat(
439             BUCKET_CACHE_COMBINED_PERCENTAGE_KEY,
440             DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE);
441         String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY);
442         int[] bucketSizes = null;
443         if (configuredBucketSizes != null) {
444           bucketSizes = new int[configuredBucketSizes.length];
445           for (int i = 0; i < configuredBucketSizes.length; i++) {
446             bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i].trim());
447           }
448         }
449         if (combinedWithLru) {
450           lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize);
451           bucketCacheSize = (long) (combinedPercentage * bucketCacheSize);
452         }
453         try {
454           int ioErrorsTolerationDuration = conf.getInt(
455               "hbase.bucketcache.ioengine.errors.tolerated.duration",
456               BucketCache.DEFAULT_ERROR_TOLERATION_DURATION);
457           bucketCache = new BucketCache(bucketCacheIOEngineName,
458               bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath,
459               ioErrorsTolerationDuration);
460         } catch (IOException ioex) {
461           LOG.error("Can't instantiate bucket cache", ioex);
462           throw new RuntimeException(ioex);
463         }
464       }
465       LOG.info("Allocating LruBlockCache with maximum size " +
466         StringUtils.humanReadableInt(lruCacheSize));
467       LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf);
468       lruCache.setVictimCache(bucketCache);
469       if (bucketCache != null && combinedWithLru) {
470         GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache);
471       } else {
472         GLOBAL_BLOCK_CACHE_INSTANCE = lruCache;
473       }
474     } else {
475       LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement.");
476       GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache(
477           lruCacheSize, offHeapCacheSize, blockSize, blockSize, conf);
478     }
479     return GLOBAL_BLOCK_CACHE_INSTANCE;
480   }
481 }