001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 * 
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 * 
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.service;
019
020import org.apache.commons.logging.LogFactory;
021import org.apache.log4j.LogManager;
022import org.apache.log4j.PropertyConfigurator;
023import org.apache.oozie.util.Instrumentable;
024import org.apache.oozie.util.Instrumentation;
025import org.apache.oozie.util.XLog;
026import org.apache.oozie.util.XLogStreamer;
027import org.apache.oozie.util.XConfiguration;
028import org.apache.oozie.BuildInfo;
029import org.apache.oozie.ErrorCode;
030import org.apache.hadoop.conf.Configuration;
031
032import java.io.File;
033import java.io.FileInputStream;
034import java.io.IOException;
035import java.io.InputStream;
036import java.io.Writer;
037import java.net.URL;
038import java.util.Properties;
039import java.util.Map;
040import java.util.Date;
041import java.util.regex.Pattern;
042
043/**
044 * Built-in service that initializes and manages Logging via Log4j.
045 * <p/>
046 * Oozie Lo4gj default configuration file is <code>oozie-log4j.properties</code>.
047 * <p/>
048 * The file name can be changed by setting the Java System property <code>oozie.log4j.file</code>.
049 * <p/>
050 * The Log4j configuration files must be a properties file.
051 * <p/>
052 * The Log4j configuration file is first looked in the Oozie configuration directory see {@link ConfigurationService}.
053 * If the file is not found there, it is looked in the classpath.
054 * <p/>
055 * If the Log4j configuration file is loaded from Oozie configuration directory, automatic reloading is enabled.
056 * <p/>
057 * If the Log4j configuration file is loaded from the classpath, automatic reloading is disabled.
058 * <p/>
059 * the automatic reloading interval is defined by the Java System property <code>oozie.log4j.reload</code>. The default
060 * value is 10 seconds.
061 */
062public class XLogService implements Service, Instrumentable {
063    private static final String INSTRUMENTATION_GROUP = "logging";
064
065    /**
066     * System property that indicates the logs directory.
067     */
068    public static final String OOZIE_LOG_DIR = "oozie.log.dir";
069
070    /**
071     * System property that indicates the log4j configuration file to load.
072     */
073    public static final String LOG4J_FILE = "oozie.log4j.file";
074
075    /**
076     * System property that indicates the reload interval of the configuration file.
077     */
078    public static final String LOG4J_RELOAD = "oozie.log4j.reload";
079
080    /**
081     * Default value for the log4j configuration file if {@link #LOG4J_FILE} is not set.
082     */
083    public static final String DEFAULT_LOG4J_PROPERTIES = "oozie-log4j.properties";
084
085    /**
086     * Default value for the reload interval if {@link #LOG4J_RELOAD} is not set.
087     */
088    public static final String DEFAULT_RELOAD_INTERVAL = "10";
089
090    private XLog log;
091    private long interval;
092    private boolean fromClasspath;
093    private String log4jFileName;
094    private boolean logOverWS = true;
095
096    private static final String STARTUP_MESSAGE = "{E}"
097            + " ******************************************************************************* {E}"
098            + "  STARTUP MSG: Oozie BUILD_VERSION [{0}] compiled by [{1}] on [{2}]{E}"
099            + "  STARTUP MSG:       revision [{3}]@[{4}]{E}"
100            + "*******************************************************************************";
101
102    private String oozieLogPath;
103    private String oozieLogName;
104    private int oozieLogRotation = -1;
105
106    public XLogService() {
107    }
108    
109    public String getOozieLogPath() {
110        return oozieLogPath;
111    }
112    
113    public String getOozieLogName() {
114        return oozieLogName;
115    }
116
117    /**
118     * Initialize the log service.
119     *
120     * @param services services instance.
121     * @throws ServiceException thrown if the log service could not be initialized.
122     */
123    public void init(Services services) throws ServiceException {
124        String oozieHome = Services.getOozieHome();
125        String oozieLogs = System.getProperty(OOZIE_LOG_DIR, oozieHome + "/logs");
126        System.setProperty(OOZIE_LOG_DIR, oozieLogs);
127        try {
128            LogManager.resetConfiguration();
129            log4jFileName = System.getProperty(LOG4J_FILE, DEFAULT_LOG4J_PROPERTIES);
130            if (log4jFileName.contains("/")) {
131                throw new ServiceException(ErrorCode.E0011, log4jFileName);
132            }
133            if (!log4jFileName.endsWith(".properties")) {
134                throw new ServiceException(ErrorCode.E0012, log4jFileName);
135            }
136            String configPath = ConfigurationService.getConfigurationDirectory();
137            File log4jFile = new File(configPath, log4jFileName);
138            if (log4jFile.exists()) {
139                fromClasspath = false;
140            }
141            else {
142                ClassLoader cl = Thread.currentThread().getContextClassLoader();
143                URL log4jUrl = cl.getResource(log4jFileName);
144                if (log4jUrl == null) {
145                    throw new ServiceException(ErrorCode.E0013, log4jFileName, configPath);
146                }
147                fromClasspath = true;
148            }
149
150            if (fromClasspath) {
151                ClassLoader cl = Thread.currentThread().getContextClassLoader();
152                URL log4jUrl = cl.getResource(log4jFileName);
153                PropertyConfigurator.configure(log4jUrl);
154            }
155            else {
156                interval = Long.parseLong(System.getProperty(LOG4J_RELOAD, DEFAULT_RELOAD_INTERVAL));
157                PropertyConfigurator.configureAndWatch(log4jFile.toString(), interval * 1000);
158            }
159
160            log = new XLog(LogFactory.getLog(getClass()));
161
162            log.info(XLog.OPS, STARTUP_MESSAGE, BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION),
163                    BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_USER_NAME), BuildInfo.getBuildInfo()
164                            .getProperty(BuildInfo.BUILD_TIME), BuildInfo.getBuildInfo().getProperty(
165                            BuildInfo.BUILD_VC_REVISION), BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VC_URL));
166
167            String from = (fromClasspath) ? "CLASSPATH" : configPath;
168            String reload = (fromClasspath) ? "disabled" : Long.toString(interval) + " sec";
169            log.info("Log4j configuration file [{0}]", log4jFileName);
170            log.info("Log4j configuration file loaded from [{0}]", from);
171            log.info("Log4j reload interval [{0}]", reload);
172
173            XLog.Info.reset();
174            XLog.Info.defineParameter(USER);
175            XLog.Info.defineParameter(GROUP);
176            XLogStreamer.Filter.reset();
177            XLogStreamer.Filter.defineParameter(USER);
178            XLogStreamer.Filter.defineParameter(GROUP);
179
180            // Getting configuration for oozie log via WS
181            ClassLoader cl = Thread.currentThread().getContextClassLoader();
182            InputStream is = (fromClasspath) ? cl.getResourceAsStream(log4jFileName) : new FileInputStream(log4jFile);
183            extractInfoForLogWebService(is);
184        }
185        catch (IOException ex) {
186            throw new ServiceException(ErrorCode.E0010, ex.getMessage(), ex);
187        }
188    }
189
190    private void extractInfoForLogWebService(InputStream is) throws IOException {
191        Properties props = new Properties();
192        props.load(is);
193
194        Configuration conf = new XConfiguration();
195        for (Map.Entry entry : props.entrySet()) {
196            conf.set((String) entry.getKey(), (String) entry.getValue());
197        }
198        String logFile = conf.get("log4j.appender.oozie.File");
199        if (logFile == null) {
200            log.warn("Oozie WS log will be disabled, missing property 'log4j.appender.oozie.File' for 'oozie' "
201                    + "appender");
202            logOverWS = false;
203        }
204        else {
205            logFile = logFile.trim();
206            int i = logFile.lastIndexOf("/");
207            if (i == -1) {
208                log.warn("Oozie WS log will be disabled, log file is not an absolute path [{0}] for 'oozie' appender",
209                        logFile);
210                logOverWS = false;
211            }
212            else {
213                String appenderClass = conf.get("log4j.appender.oozie");
214                if (appenderClass == null) {
215                    log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie]");
216                    logOverWS = false;
217                }
218                else if (appenderClass.equals("org.apache.log4j.DailyRollingFileAppender")) {
219                    String pattern = conf.get("log4j.appender.oozie.DatePattern");
220                    if (pattern == null) {
221                        log.warn("Oozie WS log will be disabled, missing property [log4j.appender.oozie.DatePattern]");
222                        logOverWS = false;
223                    }
224                    else {
225                        pattern = pattern.trim();
226                        if (pattern.endsWith("HH")) {
227                            oozieLogRotation = 60 * 60;
228                        }
229                        else if (pattern.endsWith("dd")) {
230                                oozieLogRotation = 60 * 60 * 24;
231                        }
232                        else {
233                            log.warn("Oozie WS log will be disabled, DatePattern [{0}] should end with 'HH' or 'dd'",
234                                    pattern);
235                            logOverWS = false;
236                        }
237                        if (oozieLogRotation > 0) {
238                            oozieLogPath = logFile.substring(0, i);
239                            oozieLogName = logFile.substring(i + 1);
240                        }
241                    }
242                }
243                else if (appenderClass.equals("org.apache.log4j.rolling.RollingFileAppender")) {
244                    String pattern = conf.get("log4j.appender.oozie.RollingPolicy.FileNamePattern");
245                    if (pattern == null) {
246                        log.warn("Oozie WS log will be disabled, missing property "
247                                + "[log4j.appender.oozie.RollingPolicy.FileNamePattern]");
248                        logOverWS = false;
249                    }
250                    else {
251                        pattern = pattern.trim();
252                        if (pattern.matches(Pattern.quote(logFile) + ".*-%d\\{yyyy-MM-dd-HH\\}(\\.gz)?")) {
253                            oozieLogRotation = 60 * 60;
254                        }
255                        else {
256                            log.warn("Oozie WS log will be disabled, RollingPolicy.FileNamePattern [{0}] should end with " 
257                                    + "'-%d{yyyy-MM-dd-HH}' or '-%d{yyyy-MM-dd-HH}.gz' and also start with the value of "
258                                    + "log4j.appender.oozie.File [{1}]", pattern, logFile);
259                            logOverWS = false;
260                        }
261                        if (oozieLogRotation > 0) {
262                            oozieLogPath = logFile.substring(0, i);
263                            oozieLogName = logFile.substring(i + 1);
264                        }
265                    }
266                }
267                else {
268                    log.warn("Oozie WS log will be disabled, log4j.appender.oozie [" + appenderClass + "] should be "
269                            + "either org.apache.log4j.DailyRollingFileAppender or org.apache.log4j.rolling.RollingFileAppender "
270                            + "to enable it");
271                    logOverWS = false;
272                }
273            }
274        }
275    }
276
277    /**
278     * Destroy the log service.
279     */
280    public void destroy() {
281        LogManager.shutdown();
282        XLog.Info.reset();
283        XLogStreamer.Filter.reset();
284    }
285
286    /**
287     * Group log info constant.
288     */
289    public static final String USER = "USER";
290
291    /**
292     * Group log info constant.
293     */
294    public static final String GROUP = "GROUP";
295
296    /**
297     * Return the public interface for log service.
298     *
299     * @return {@link XLogService}.
300     */
301    public Class<? extends Service> getInterface() {
302        return XLogService.class;
303    }
304
305    /**
306     * Instruments the log service.
307     * <p/>
308     * It sets instrumentation variables indicating the config file, reload interval and if loaded from the classpath.
309     *
310     * @param instr instrumentation to use.
311     */
312    public void instrument(Instrumentation instr) {
313        instr.addVariable("oozie", "version", new Instrumentation.Variable<String>() {
314            public String getValue() {
315                return BuildInfo.getBuildInfo().getProperty(BuildInfo.BUILD_VERSION);
316            }
317        });
318        instr.addVariable(INSTRUMENTATION_GROUP, "config.file", new Instrumentation.Variable<String>() {
319            public String getValue() {
320                return log4jFileName;
321            }
322        });
323        instr.addVariable(INSTRUMENTATION_GROUP, "reload.interval", new Instrumentation.Variable<Long>() {
324            public Long getValue() {
325                return interval;
326            }
327        });
328        instr.addVariable(INSTRUMENTATION_GROUP, "from.classpath", new Instrumentation.Variable<Boolean>() {
329            public Boolean getValue() {
330                return fromClasspath;
331            }
332        });
333        instr.addVariable(INSTRUMENTATION_GROUP, "log.over.web-service", new Instrumentation.Variable<Boolean>() {
334            public Boolean getValue() {
335                return logOverWS;
336            }
337        });
338    }
339
340    /**
341     * Stream the log of a job.
342     *
343     * @param filter log streamer filter.
344     * @param startTime start time for log events to filter.
345     * @param endTime end time for log events to filter.
346     * @param writer writer to stream the log to.
347     * @throws IOException thrown if the log cannot be streamed.
348     */
349    public void streamLog(XLogStreamer.Filter filter, Date startTime, Date endTime, Writer writer) throws IOException {
350        if (logOverWS) {
351            new XLogStreamer(filter, writer, oozieLogPath, oozieLogName, oozieLogRotation)
352                    .streamLog(startTime, endTime);
353        }
354        else {
355            writer.write("Log streaming disabled!!");
356        }
357
358    }
359
360    String getLog4jProperties() {
361        return log4jFileName;
362    }
363
364    boolean getFromClasspath() {
365        return fromClasspath;
366    }
367
368}