1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.DataInput;
23 import java.io.IOException;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.classification.InterfaceAudience;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
31 import org.apache.hadoop.hbase.io.hfile.HFile;
32 import org.apache.hadoop.hbase.regionserver.StoreFile;
33 import org.apache.hadoop.hbase.regionserver.BloomType;
34
35
36
37
38
39 @InterfaceAudience.Private
40 public final class BloomFilterFactory {
41
42 private static final Log LOG =
43 LogFactory.getLog(BloomFilterFactory.class.getName());
44
45
46 private BloomFilterFactory() {}
47
48
49
50
51
52 public static final String IO_STOREFILE_BLOOM_ERROR_RATE =
53 "io.storefile.bloom.error.rate";
54
55
56
57
58
59 public static final String IO_STOREFILE_BLOOM_MAX_FOLD =
60 "io.storefile.bloom.max.fold";
61
62
63
64
65
66 public static final String IO_STOREFILE_BLOOM_MAX_KEYS =
67 "io.storefile.bloom.max.keys";
68
69
70 public static final String IO_STOREFILE_BLOOM_ENABLED =
71 "io.storefile.bloom.enabled";
72
73
74 public static final String IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED =
75 "io.storefile.delete.family.bloom.enabled";
76
77
78
79
80
81 public static final String IO_STOREFILE_BLOOM_BLOCK_SIZE =
82 "io.storefile.bloom.block.size";
83
84
85 private static final int MAX_ALLOWED_FOLD_FACTOR = 7;
86
87
88
89
90
91
92
93
94
95
96
97
98 public static BloomFilter
99 createFromMeta(DataInput meta, HFile.Reader reader)
100 throws IllegalArgumentException, IOException {
101 int version = meta.readInt();
102 switch (version) {
103 case ByteBloomFilter.VERSION:
104
105
106
107 return new ByteBloomFilter(meta);
108
109 case CompoundBloomFilterBase.VERSION:
110 return new CompoundBloomFilter(meta, reader);
111
112 default:
113 throw new IllegalArgumentException(
114 "Bad bloom filter format version " + version
115 );
116 }
117 }
118
119
120
121
122
123 public static boolean isGeneralBloomEnabled(Configuration conf) {
124 return conf.getBoolean(IO_STOREFILE_BLOOM_ENABLED, true);
125 }
126
127
128
129
130 public static boolean isDeleteFamilyBloomEnabled(Configuration conf) {
131 return conf.getBoolean(IO_STOREFILE_DELETEFAMILY_BLOOM_ENABLED, true);
132 }
133
134
135
136
137 public static float getErrorRate(Configuration conf) {
138 return conf.getFloat(IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01);
139 }
140
141
142
143
144 public static int getMaxFold(Configuration conf) {
145 return conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD, MAX_ALLOWED_FOLD_FACTOR);
146 }
147
148
149 public static int getBloomBlockSize(Configuration conf) {
150 return conf.getInt(IO_STOREFILE_BLOOM_BLOCK_SIZE, 128 * 1024);
151 }
152
153
154
155
156 public static int getMaxKeys(Configuration conf) {
157 return conf.getInt(IO_STOREFILE_BLOOM_MAX_KEYS, 128 * 1000 * 1000);
158 }
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173 public static BloomFilterWriter createGeneralBloomAtWrite(Configuration conf,
174 CacheConfig cacheConf, BloomType bloomType, int maxKeys,
175 HFile.Writer writer) {
176 if (!isGeneralBloomEnabled(conf)) {
177 LOG.trace("Bloom filters are disabled by configuration for "
178 + writer.getPath()
179 + (conf == null ? " (configuration is null)" : ""));
180 return null;
181 } else if (bloomType == BloomType.NONE) {
182 LOG.trace("Bloom filter is turned off for the column family");
183 return null;
184 }
185
186 float err = getErrorRate(conf);
187
188
189
190
191
192 if (bloomType == BloomType.ROWCOL) {
193 err = (float) (1 - Math.sqrt(1 - err));
194 }
195
196 int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD,
197 MAX_ALLOWED_FOLD_FACTOR);
198
199
200
201 CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
202 err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
203 bloomType == BloomType.ROWCOL ? KeyValue.COMPARATOR : KeyValue.RAW_COMPARATOR);
204 writer.addInlineBlockWriter(bloomWriter);
205 return bloomWriter;
206 }
207
208
209
210
211
212
213
214
215
216
217
218
219 public static BloomFilterWriter createDeleteBloomAtWrite(Configuration conf,
220 CacheConfig cacheConf, int maxKeys, HFile.Writer writer) {
221 if (!isDeleteFamilyBloomEnabled(conf)) {
222 LOG.info("Delete Bloom filters are disabled by configuration for "
223 + writer.getPath()
224 + (conf == null ? " (configuration is null)" : ""));
225 return null;
226 }
227
228 float err = getErrorRate(conf);
229
230 int maxFold = getMaxFold(conf);
231
232 CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
233 err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
234 KeyValue.RAW_COMPARATOR);
235 writer.addInlineBlockWriter(bloomWriter);
236 return bloomWriter;
237 }
238 };