001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * 
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 * 
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.service;
019
020import org.apache.hadoop.conf.Configuration;
021import org.apache.oozie.util.Instrumentable;
022import org.apache.oozie.util.Instrumentation;
023import org.apache.oozie.util.XLog;
024import org.apache.oozie.util.XConfiguration;
025import org.apache.oozie.ErrorCode;
026
027import java.io.File;
028import java.io.FileInputStream;
029import java.io.IOException;
030import java.io.InputStream;
031import java.io.StringWriter;
032import java.util.HashSet;
033import java.util.Map;
034import java.util.Set;
035import java.util.Arrays;
036
037/**
038 * Built in service that initializes the services configuration.
039 * <p/>
040 * The configuration loading sequence is identical to Hadoop configuration loading sequence.
041 * <p/>
042 * Default values are loaded from the 'oozie-default.xml' file from the classpath, then site configured values
043 * are loaded from a site configuration file from the Oozie configuration directory.
044 * <p/>
045 * The Oozie configuration directory is resolved using the <code>OOZIE_HOME<code> environment variable as
046 * <code>${OOZIE_HOME}/conf</code>. If the <code>OOZIE_HOME<code> environment variable is not defined the
047 * initialization of the <code>ConfigurationService</code> fails.
048 * <p/>
049 * The site configuration is loaded from the <code>oozie-site.xml</code> file in the configuration directory.
050 * <p/>
051 * The site configuration file name to use can be changed by setting the <code>OOZIE_CONFIG_FILE</code> environment
052 * variable to an alternate file name. The alternate file must ber in the Oozie configuration directory.
053 * <p/>
054 * Configuration properties, prefixed with 'oozie.', passed as system properties overrides default and site values.
055 * <p/>
056 * The configuration service logs details on how the configuration was loaded as well as what properties were overrode
057 * via system properties settings.
058 */
059public class ConfigurationService implements Service, Instrumentable {
060    private static final String INSTRUMENTATION_GROUP = "configuration";
061
062    public static final String CONF_PREFIX = Service.CONF_PREFIX + "ConfigurationService.";
063
064    public static final String CONF_IGNORE_SYS_PROPS = CONF_PREFIX + "ignore.system.properties";
065
066    /**
067     * System property that indicates the configuration directory.
068     */
069    public static final String OOZIE_CONFIG_DIR = "oozie.config.dir";
070
071
072    /**
073     * System property that indicates the data directory.
074     */
075    public static final String OOZIE_DATA_DIR = "oozie.data.dir";
076
077    /**
078     * System property that indicates the name of the site configuration file to load.
079     */
080    public static final String OOZIE_CONFIG_FILE = "oozie.config.file";
081
082    private static final Set<String> IGNORE_SYS_PROPS = new HashSet<String>();
083    private static final String IGNORE_TEST_SYS_PROPS = "oozie.test.";
084
085    private static final String PASSWORD_PROPERTY_END = ".password";
086
087    static {
088        IGNORE_SYS_PROPS.add(CONF_IGNORE_SYS_PROPS);
089
090        //all this properties are seeded as system properties, no need to log changes
091        IGNORE_SYS_PROPS.add("oozie.http.hostname");
092        IGNORE_SYS_PROPS.add("oozie.http.port");
093
094        IGNORE_SYS_PROPS.add(Services.OOZIE_HOME_DIR);
095        IGNORE_SYS_PROPS.add(OOZIE_CONFIG_DIR);
096        IGNORE_SYS_PROPS.add(OOZIE_CONFIG_FILE);
097        IGNORE_SYS_PROPS.add(OOZIE_DATA_DIR);
098
099        IGNORE_SYS_PROPS.add(XLogService.OOZIE_LOG_DIR);
100        IGNORE_SYS_PROPS.add(XLogService.LOG4J_FILE);
101        IGNORE_SYS_PROPS.add(XLogService.LOG4J_RELOAD);
102    }
103
104    public static final String DEFAULT_CONFIG_FILE = "oozie-default.xml";
105    public static final String SITE_CONFIG_FILE = "oozie-site.xml";
106
107    private static XLog log = XLog.getLog(ConfigurationService.class);
108
109    private String configDir;
110    private String configFile;
111
112    private LogChangesConfiguration configuration;
113
114    public ConfigurationService() {
115        log = XLog.getLog(ConfigurationService.class);
116    }
117
118    /**
119     * Initialize the log service.
120     *
121     * @param services services instance.
122     * @throws ServiceException thrown if the log service could not be initialized.
123     */
124    public void init(Services services) throws ServiceException {
125        configDir = getConfigurationDirectory();
126        configFile = System.getProperty(OOZIE_CONFIG_FILE, SITE_CONFIG_FILE);
127        if (configFile.contains("/")) {
128            throw new ServiceException(ErrorCode.E0022, configFile);
129        }
130        log.info("Oozie home dir  [{0}]", Services.getOozieHome());
131        log.info("Oozie conf dir  [{0}]", configDir);
132        log.info("Oozie conf file [{0}]", configFile);
133        configFile = new File(configDir, configFile).toString();
134        configuration = loadConf();
135    }
136
137    public static String getConfigurationDirectory() throws ServiceException {
138        String oozieHome = Services.getOozieHome();
139        String configDir = System.getProperty(OOZIE_CONFIG_DIR, oozieHome + "/conf");
140        File file = new File(configDir);
141        if (!file.exists()) {
142            throw new ServiceException(ErrorCode.E0024, configDir);
143        }
144        return configDir;
145    }
146
147    /**
148     * Destroy the configuration service.
149     */
150    public void destroy() {
151        configuration = null;
152    }
153
154    /**
155     * Return the public interface for configuration service.
156     *
157     * @return {@link ConfigurationService}.
158     */
159    public Class<? extends Service> getInterface() {
160        return ConfigurationService.class;
161    }
162
163    /**
164     * Return the services configuration.
165     *
166     * @return the services configuration.
167     */
168    public Configuration getConf() {
169        if (configuration == null) {
170            throw new IllegalStateException("Not initialized");
171        }
172        return configuration;
173    }
174
175    /**
176     * Return Oozie configuration directory.
177     *
178     * @return Oozie configuration directory.
179     */
180    public String getConfigDir() {
181        return configDir;
182    }
183
184    private InputStream getDefaultConfiguration() throws ServiceException, IOException {
185        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
186        InputStream inputStream = classLoader.getResourceAsStream(DEFAULT_CONFIG_FILE);
187        if (inputStream == null) {
188            throw new ServiceException(ErrorCode.E0023, DEFAULT_CONFIG_FILE);
189        }
190        return inputStream;
191    }
192
193    private LogChangesConfiguration loadConf() throws ServiceException {
194        XConfiguration configuration;
195        try {
196            InputStream inputStream = getDefaultConfiguration();
197            configuration = new XConfiguration(inputStream);
198            File file = new File(configFile);
199            if (!file.exists()) {
200                log.info("Missing site configuration file [{0}]", configFile);
201            }
202            else {
203                inputStream = new FileInputStream(configFile);
204                XConfiguration siteConfiguration = new XConfiguration(inputStream);
205                XConfiguration.injectDefaults(configuration, siteConfiguration);
206                configuration = siteConfiguration;
207            }
208        }
209        catch (IOException ex) {
210            throw new ServiceException(ErrorCode.E0024, configFile, ex.getMessage(), ex);
211        }
212
213        if (log.isTraceEnabled()) {
214            try {
215                StringWriter writer = new StringWriter();
216                for (Map.Entry<String, String> entry : configuration) {
217                    boolean maskValue = entry.getKey().endsWith(PASSWORD_PROPERTY_END);
218                    String value = (maskValue) ? "**MASKED**" : entry.getValue();
219                    writer.write(" " + entry.getKey() + " = " + value + "\n");
220                }
221                writer.close();
222                log.trace("Configuration:\n{0}---", writer.toString());
223            }
224            catch (IOException ex) {
225                throw new ServiceException(ErrorCode.E0025, ex.getMessage(), ex);
226            }
227        }
228
229        String[] ignoreSysProps = configuration.getStrings(CONF_IGNORE_SYS_PROPS);
230        if (ignoreSysProps != null) {
231            IGNORE_SYS_PROPS.addAll(Arrays.asList(ignoreSysProps));
232        }
233
234        for (Map.Entry<String, String> entry : configuration) {
235            String sysValue = System.getProperty(entry.getKey());
236            if (sysValue != null && !IGNORE_SYS_PROPS.contains(entry.getKey())) {
237                log.info("Configuration change via System Property, [{0}]=[{1}]", entry.getKey(), sysValue);
238                configuration.set(entry.getKey(), sysValue);
239            }
240        }
241        for (Map.Entry<Object, Object> entry : System.getProperties().entrySet()) {
242            String name = (String) entry.getKey();
243            if (!IGNORE_SYS_PROPS.contains(name)) {
244                if (name.startsWith("oozie.") && !name.startsWith(IGNORE_TEST_SYS_PROPS)) {
245                    if (configuration.get(name) == null) {
246                        log.warn("System property [{0}] no defined in Oozie configuration, ignored", name);
247                    }
248                }
249            }
250        }
251
252        return new LogChangesConfiguration(configuration);
253    }
254
255    private class LogChangesConfiguration extends XConfiguration {
256
257        public LogChangesConfiguration(Configuration conf) {
258            for (Map.Entry<String, String> entry : conf) {
259                if (get(entry.getKey()) == null) {
260                    setValue(entry.getKey(), entry.getValue());
261                }
262            }
263        }
264
265        public String[] getStrings(String name) {
266            String s = get(name);
267            return (s != null && s.trim().length() > 0) ? super.getStrings(name) : new String[0];
268        }
269
270        public String get(String name, String defaultValue) {
271            String value = get(name);
272            if (value == null) {
273                boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
274                value = defaultValue;
275                String logValue = (maskValue) ? "**MASKED**" : defaultValue;
276                log.warn(XLog.OPS, "Configuration property [{0}] not found, using default [{1}]", name, logValue);
277            }
278            return value;
279        }
280
281        public void set(String name, String value) {
282            setValue(name, value);
283            boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
284            value = (maskValue) ? "**MASKED**" : value;
285            log.info(XLog.OPS, "Programmatic configuration change, property[{0}]=[{1}]", name, value);
286        }
287
288        private void setValue(String name, String value) {
289            super.set(name, value);
290        }
291
292    }
293
294    /**
295     * Instruments the configuration service. <p/> It sets instrumentation variables indicating the config dir and
296     * config file used.
297     *
298     * @param instr instrumentation to use.
299     */
300    public void instrument(Instrumentation instr) {
301        instr.addVariable(INSTRUMENTATION_GROUP, "config.dir", new Instrumentation.Variable<String>() {
302            public String getValue() {
303                return configDir;
304            }
305        });
306        instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
307            public String getValue() {
308                return configFile;
309            }
310        });
311        instr.setConfiguration(configuration);
312    }
313
314    /**
315     * Return a configuration with all sensitive values masked.
316     *
317     * @param conf configuration to mask.
318     * @return masked configuration.
319     */
320    public static Configuration maskPasswords(Configuration conf) {
321        XConfiguration maskedConf = new XConfiguration();
322        for (Map.Entry<String, String> entry : conf) {
323            String name = entry.getKey();
324            boolean maskValue = name.endsWith(PASSWORD_PROPERTY_END);
325            String value = (maskValue) ? "**MASKED**" : entry.getValue();
326            maskedConf.set(name, value);
327        }
328        return maskedConf;
329    }
330
331}