001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *      http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.oozie.coord;
019
020import java.util.Calendar;
021import java.util.Date;
022import java.util.HashMap;
023import java.util.Iterator;
024import java.util.List;
025import java.util.Map;
026
027import org.apache.hadoop.conf.Configuration;
028import org.apache.oozie.command.coord.CoordCommandUtils;
029import org.apache.oozie.service.ELService;
030import org.apache.oozie.service.Services;
031import org.apache.oozie.util.DateUtils;
032import org.apache.oozie.util.ELEvaluator;
033import org.apache.oozie.util.XmlUtils;
034import org.jdom.Element;
035
036/**
037 * This class provide different evaluators required at different stages
038 */
039public class CoordELEvaluator {
040    public static final Integer MINUTE = 1;
041    public static final Integer HOUR = 60 * MINUTE;
042
043    /**
044     * Create an evaluator to be used in resolving configuration vars and frequency constant/functions (used in Stage
045     * 1)
046     *
047     * @param conf : Configuration containing property variables
048     * @return configured ELEvaluator
049     */
050    public static ELEvaluator createELEvaluatorForGroup(Configuration conf, String group) {
051        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(group);
052        setConfigToEval(eval, conf);
053        return eval;
054    }
055
056    /**
057     * Create a new Evaluator to resolve the EL functions and variables using action creation time (Phase 2)
058     *
059     * @param event : Xml element for data-in element usually enclosed by <data-in(out)> tag
060     * @param appInst : Application Instance related information such as Action creation Time
061     * @param conf :Configuration to substitute any variables
062     * @return configured ELEvaluator
063     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
064     */
065    public static ELEvaluator createInstancesELEvaluator(Element event, SyncCoordAction appInst, Configuration conf)
066            throws Exception {
067        return createInstancesELEvaluator("coord-action-create", event, appInst, conf);
068    }
069
070    public static ELEvaluator createInstancesELEvaluator(String tag, Element event, SyncCoordAction appInst,
071                                                         Configuration conf) throws Exception {
072        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator(tag);
073        setConfigToEval(eval, conf);
074        SyncCoordDataset ds = getDSObject(event);
075        CoordELFunctions.configureEvaluator(eval, ds, appInst);
076        return eval;
077    }
078
079    public static ELEvaluator createELEvaluatorForDataEcho(Configuration conf, String group,
080                                                           HashMap<String, String> dataNameList) throws Exception {
081        ELEvaluator eval = createELEvaluatorForGroup(conf, group);
082        for (Iterator<String> it = dataNameList.keySet().iterator(); it.hasNext();) {
083            String key = it.next();
084            String value = dataNameList.get(key);
085            eval.setVariable("oozie.dataname." + key, value);
086        }
087        return eval;
088    }
089
090    /**
091     * Create a new evaluator for Lazy resolve (phase 3). For example, coord_latest(n) and coord_actualTime()function
092     * should be resolved when all other data dependencies are met.
093     *
094     * @param actualTime : Action start time
095     * @param nominalTime : Action creation time
096     * @param dEvent :XML element for data-in element usually enclosed by <data-in(out)> tag
097     * @param conf :Configuration to substitute any variables
098     * @return configured ELEvaluator
099     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
100     */
101    public static ELEvaluator createLazyEvaluator(Date actualTime, Date nominalTime, Element dEvent, Configuration conf)
102            throws Exception {
103        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-action-start");
104        setConfigToEval(eval, conf);
105        SyncCoordDataset ds = getDSObject(dEvent);
106        SyncCoordAction appInst = new SyncCoordAction();
107        appInst.setNominalTime(nominalTime);
108        appInst.setActualTime(actualTime);
109        CoordELFunctions.configureEvaluator(eval, ds, appInst);
110        eval.setVariable(CoordELFunctions.CONFIGURATION, conf);
111        return eval;
112    }
113
114    /**
115     * Create a SLA evaluator to be used during Materialization
116     *
117     * @param nominalTime
118     * @param conf
119     * @return
120     * @throws Exception
121     */
122    public static ELEvaluator createSLAEvaluator(Date nominalTime, Configuration conf) throws Exception {
123        ELEvaluator eval = Services.get().get(ELService.class).createEvaluator("coord-sla-create");
124        setConfigToEval(eval, conf);
125        SyncCoordAction appInst = new SyncCoordAction();// TODO:
126        appInst.setNominalTime(nominalTime);
127        CoordELFunctions.configureEvaluator(eval, null, appInst);
128        return eval;
129    }
130
131    /**
132     * Create an Evaluator to resolve dataIns and dataOuts of an application instance (used in stage 3)
133     *
134     * @param eJob : XML element for the application instance
135     * @param conf :Configuration to substitute any variables
136     * @return configured ELEvaluator
137     * @throws Exception : If there is any date-time string in wrong format, the exception is thrown
138     */
139    public static ELEvaluator createDataEvaluator(Element eJob, Configuration conf, String actionId) throws Exception {
140        ELEvaluator e = Services.get().get(ELService.class).createEvaluator("coord-action-start");
141        setConfigToEval(e, conf);
142        SyncCoordAction appInst = new SyncCoordAction();
143        String strNominalTime = eJob.getAttributeValue("action-nominal-time");
144        if (strNominalTime != null) {
145            appInst.setNominalTime(DateUtils.parseDateOozieTZ(strNominalTime));
146            appInst.setTimeZone(DateUtils.getTimeZone(eJob.getAttributeValue("timezone")));
147            appInst.setFrequency(Integer.parseInt(eJob.getAttributeValue("frequency")));
148            appInst.setTimeUnit(TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit")));
149            appInst.setActionId(actionId);
150            appInst.setName(eJob.getAttributeValue("name"));
151        }
152        String strActualTime = eJob.getAttributeValue("action-actual-time");
153        if (strActualTime != null) {
154            appInst.setActualTime(DateUtils.parseDateOozieTZ(strActualTime));
155        }
156        CoordELFunctions.configureEvaluator(e, null, appInst);
157        Element events = eJob.getChild("input-events", eJob.getNamespace());
158        if (events != null) {
159            for (Element data : (List<Element>) events.getChildren("data-in", eJob.getNamespace())) {
160                if (data.getChild("uris", data.getNamespace()) != null) {
161                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
162                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
163                    e.setVariable(".datain." + data.getAttributeValue("name"), uris);
164                }
165                else {
166                }
167                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
168                    e.setVariable(".datain." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
169                    // check
170                    // null
171                }
172            }
173        }
174        events = eJob.getChild("output-events", eJob.getNamespace());
175        if (events != null) {
176            for (Element data : (List<Element>) events.getChildren("data-out", eJob.getNamespace())) {
177                if (data.getChild("uris", data.getNamespace()) != null) {
178                    String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
179                    uris = uris.replaceAll(CoordELFunctions.INSTANCE_SEPARATOR, CoordELFunctions.DIR_SEPARATOR);
180                    e.setVariable(".dataout." + data.getAttributeValue("name"), uris);
181                }
182                else {
183                }// TODO
184                if (data.getChild(CoordCommandUtils.UNRESOLVED_INST_TAG, data.getNamespace()) != null) {
185                    e.setVariable(".dataout." + data.getAttributeValue("name") + ".unresolved", "true"); // TODO:
186                    // check
187                    // null
188                }
189            }
190        }
191        return e;
192    }
193
194    /**
195     * Create a new Evaluator to resolve URI temple with time specific constant
196     *
197     * @param strDate : Date-time
198     * @return configured ELEvaluator
199     * @throws Exception If there is any date-time string in wrong format, the exception is thrown
200     */
201    public static ELEvaluator createURIELEvaluator(String strDate) throws Exception {
202        ELEvaluator eval = new ELEvaluator();
203        Calendar date = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
204        // always???
205        date.setTime(DateUtils.parseDateOozieTZ(strDate));
206        eval.setVariable("YEAR", date.get(Calendar.YEAR));
207        eval.setVariable("MONTH", make2Digits(date.get(Calendar.MONTH) + 1));
208        eval.setVariable("DAY", make2Digits(date.get(Calendar.DAY_OF_MONTH)));
209        eval.setVariable("HOUR", make2Digits(date.get(Calendar.HOUR_OF_DAY)));
210        eval.setVariable("MINUTE", make2Digits(date.get(Calendar.MINUTE)));
211        return eval;
212    }
213
214    /**
215     * Create Dataset object using the Dataset XML information
216     *
217     * @param eData
218     * @return
219     * @throws Exception
220     */
221    private static SyncCoordDataset getDSObject(Element eData) throws Exception {
222        SyncCoordDataset ds = new SyncCoordDataset();
223        Element eDataset = eData.getChild("dataset", eData.getNamespace());
224        // System.out.println("eDATA :"+ XmlUtils.prettyPrint(eData));
225        Date initInstance = DateUtils.parseDateOozieTZ(eDataset.getAttributeValue("initial-instance"));
226        ds.setInitInstance(initInstance);
227        if (eDataset.getAttributeValue("frequency") != null) {
228            int frequency = Integer.parseInt(eDataset.getAttributeValue("frequency"));
229            ds.setFrequency(frequency);
230            ds.setType("SYNC");
231            if (eDataset.getAttributeValue("freq_timeunit") == null) {
232                throw new RuntimeException("No freq_timeunit defined in data set definition\n"
233                        + XmlUtils.prettyPrint(eDataset));
234            }
235            ds.setTimeUnit(TimeUnit.valueOf(eDataset.getAttributeValue("freq_timeunit")));
236            if (eDataset.getAttributeValue("timezone") == null) {
237                throw new RuntimeException("No timezone defined in data set definition\n"
238                        + XmlUtils.prettyPrint(eDataset));
239            }
240            ds.setTimeZone(DateUtils.getTimeZone(eDataset.getAttributeValue("timezone")));
241            if (eDataset.getAttributeValue("end_of_duration") == null) {
242                throw new RuntimeException("No end_of_duration defined in data set definition\n"
243                        + XmlUtils.prettyPrint(eDataset));
244            }
245            ds.setEndOfDuration(TimeUnit.valueOf(eDataset.getAttributeValue("end_of_duration")));
246
247            Element doneFlagElement = eDataset.getChild("done-flag", eData.getNamespace());
248            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
249            ds.setDoneFlag(doneFlag);
250        }
251        else {
252            ds.setType("ASYNC");
253        }
254        String name = eDataset.getAttributeValue("name");
255        ds.setName(name);
256        // System.out.println(name + " VAL "+ eDataset.getChild("uri-template",
257        // eData.getNamespace()));
258        String uriTemplate = eDataset.getChild("uri-template", eData.getNamespace()).getTextTrim();
259        ds.setUriTemplate(uriTemplate);
260        // ds.setTimeUnit(TimeUnit.MINUTES);
261        return ds;
262    }
263
264    /**
265     * Set all job configurations properties into evaluator.
266     *
267     * @param eval : Evaluator to set variables
268     * @param conf : configurations to set Evaluator
269     */
270    private static void setConfigToEval(ELEvaluator eval, Configuration conf) {
271        for (Map.Entry<String, String> entry : conf) {
272            eval.setVariable(entry.getKey(), entry.getValue().trim());
273        }
274    }
275
276    /**
277     * make any one digit number to two digit string pre-appending a"0"
278     *
279     * @param num : number to make sting
280     * @return :String of length at least two digit.
281     */
282    private static String make2Digits(int num) {
283        String ret = "" + num;
284        if (num <= 9) {
285            ret = "0" + ret;
286        }
287        return ret;
288    }
289}