View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  import java.io.IOException;
21  import java.lang.reflect.InvocationTargetException;
22  import java.lang.reflect.Method;
23  import java.util.Map.Entry;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.classification.InterfaceStability;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.util.VersionInfo;
31  
32  /**
33   * Adds HBase configuration files to a Configuration
34   */
35  @InterfaceAudience.Public
36  @InterfaceStability.Stable
37  public class HBaseConfiguration extends Configuration {
38  
39    private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
40  
41    // a constant to convert a fraction to a percentage
42    private static final int CONVERT_TO_PERCENTAGE = 100;
43  
44    /**
45     * Instantinating HBaseConfiguration() is deprecated. Please use
46     * HBaseConfiguration#create() to construct a plain Configuration
47     */
48    @Deprecated
49    public HBaseConfiguration() {
50      //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration
51      super();
52      addHbaseResources(this);
53      LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use"
54          + " HBaseConfiguration#create() to construct a plain Configuration");
55    }
56  
57    /**
58     * Instantiating HBaseConfiguration() is deprecated. Please use
59     * HBaseConfiguration#create(conf) to construct a plain Configuration
60     */
61    @Deprecated
62    public HBaseConfiguration(final Configuration c) {
63      //TODO:replace with private constructor
64      this();
65      merge(this, c);
66    }
67  
68    private static void checkDefaultsVersion(Configuration conf) {
69      if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return;
70      String defaultsVersion = conf.get("hbase.defaults.for.version");
71      String thisVersion = VersionInfo.getVersion();
72      if (!thisVersion.equals(defaultsVersion)) {
73        throw new RuntimeException(
74          "hbase-default.xml file seems to be for and old version of HBase (" +
75          defaultsVersion + "), this version is " + thisVersion);
76      }
77    }
78  
79    private static void checkForClusterFreeMemoryLimit(Configuration conf) {
80        float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f);
81        int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE);
82        float blockCacheUpperLimit =
83          conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
84            HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
85        int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
86        if (CONVERT_TO_PERCENTAGE - (gml + bcul)
87                < (int)(CONVERT_TO_PERCENTAGE *
88                        HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
89            throw new RuntimeException(
90              "Current heap configuration for MemStore and BlockCache exceeds " +
91              "the threshold required for successful cluster operation. " +
92              "The combined value cannot exceed 0.8. Please check " +
93              "the settings for hbase.regionserver.global.memstore.upperLimit and " +
94              "hfile.block.cache.size in your configuration. " +
95              "hbase.regionserver.global.memstore.upperLimit is " +
96              globalMemstoreLimit +
97              " hfile.block.cache.size is " + blockCacheUpperLimit);
98        }
99    }
100 
101   public static Configuration addHbaseResources(Configuration conf) {
102     conf.addResource("hbase-default.xml");
103     conf.addResource("hbase-site.xml");
104 
105     checkDefaultsVersion(conf);
106     checkForClusterFreeMemoryLimit(conf);
107     return conf;
108   }
109 
110   /**
111    * Creates a Configuration with HBase resources
112    * @return a Configuration with HBase resources
113    */
114   public static Configuration create() {
115     Configuration conf = new Configuration();
116     // In case HBaseConfiguration is loaded from a different classloader than
117     // Configuration, conf needs to be set with appropriate class loader to resolve
118     // HBase resources.
119     conf.setClassLoader(HBaseConfiguration.class.getClassLoader());
120     return addHbaseResources(conf);
121   }
122 
123   /**
124    * @param that Configuration to clone.
125    * @return a Configuration created with the hbase-*.xml files plus
126    * the given configuration.
127    */
128   public static Configuration create(final Configuration that) {
129     Configuration conf = create();
130     merge(conf, that);
131     return conf;
132   }
133 
134   /**
135    * Merge two configurations.
136    * @param destConf the configuration that will be overwritten with items
137    *                 from the srcConf
138    * @param srcConf the source configuration
139    **/
140   public static void merge(Configuration destConf, Configuration srcConf) {
141     for (Entry<String, String> e : srcConf) {
142       destConf.set(e.getKey(), e.getValue());
143     }
144   }
145 
146   /**
147    * @return whether to show HBase Configuration in servlet
148    */
149   public static boolean isShowConfInServlet() {
150     boolean isShowConf = false;
151     try {
152       if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) {
153         isShowConf = true;
154       }
155     } catch (LinkageError e) {
156        // should we handle it more aggressively in addition to log the error?
157        LOG.warn("Error thrown: ", e);
158     } catch (ClassNotFoundException ce) {
159       LOG.debug("ClassNotFound: ConfServlet");
160       // ignore
161     }
162     return isShowConf;
163   }
164 
165   /**
166    * Get the value of the <code>name</code> property as an <code>int</code>, possibly
167    * referring to the deprecated name of the configuration property.
168    * If no such property exists, the provided default value is returned,
169    * or if the specified value is not a valid <code>int</code>,
170    * then an error is thrown.
171    *
172    * @param name property name.
173    * @param deprecatedName a deprecatedName for the property to use
174    * if non-deprecated name is not used
175    * @param defaultValue default value.
176    * @throws NumberFormatException when the value is invalid
177    * @return property value as an <code>int</code>,
178    *         or <code>defaultValue</code>.
179    */
180   // TODO: developer note: This duplicates the functionality of deprecated
181   // property support in Configuration in Hadoop 2. But since Hadoop-1 does not
182   // contain these changes, we will do our own as usual. Replace these when H2 is default.
183   public static int getInt(Configuration conf, String name,
184       String deprecatedName, int defaultValue) {
185     if (conf.get(deprecatedName) != null) {
186       LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\""
187         , deprecatedName, name));
188       return conf.getInt(deprecatedName, defaultValue);
189     } else {
190       return conf.getInt(name, defaultValue);
191     }
192   }
193 
194   /**
195    * Get the password from the Configuration instance using the
196    * getPassword method if it exists. If not, then fall back to the
197    * general get method for configuration elements.
198    * @param conf configuration instance for accessing the passwords
199    * @param alias the name of the password element
200    * @param defPass the default password
201    * @return String password or default password
202    * @throws IOException
203    */
204   public static String getPassword(Configuration conf, String alias,
205       String defPass) throws IOException {
206     String passwd = null;
207     try {
208       Method m = Configuration.class.getMethod("getPassword", String.class);
209       char[] p = (char[]) m.invoke(conf, alias);
210       if (p != null) {
211         LOG.debug(String.format("Config option \"%s\" was found through" +
212         		" the Configuration getPassword method.", alias));
213         passwd = new String(p);
214       }
215       else {
216         LOG.debug(String.format(
217             "Config option \"%s\" was not found. Using provided default value",
218             alias));
219         passwd = defPass;
220       }
221     } catch (NoSuchMethodException e) {
222       // this is a version of Hadoop where the credential
223       //provider API doesn't exist yet
224       LOG.debug(String.format(
225           "Credential.getPassword method is not available." +
226           " Falling back to configuration."));
227       passwd = conf.get(alias, defPass);
228     } catch (SecurityException e) {
229       throw new IOException(e.getMessage(), e);
230     } catch (IllegalAccessException e) {
231       throw new IOException(e.getMessage(), e);
232     } catch (IllegalArgumentException e) {
233       throw new IOException(e.getMessage(), e);
234     } catch (InvocationTargetException e) {
235       throw new IOException(e.getMessage(), e);
236     }
237     return passwd;
238   }
239 
240   /** For debugging.  Dump configurations to system output as xml format.
241    * Master and RS configurations can also be dumped using
242    * http services. e.g. "curl http://master:60010/dump"
243    */
244   public static void main(String[] args) throws Exception {
245     HBaseConfiguration.create().writeXml(System.out);
246   }
247 }