View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.HashMap;
24  import java.util.Iterator;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.Map.Entry;
28  import java.util.Set;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.classification.InterfaceAudience;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.KeyValue.Type;
36  import org.apache.hadoop.hbase.Tag;
37  import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.apache.hadoop.hbase.util.Pair;
40  
41  /**
42   * Similar to ScanDeletTracker but tracks the visibility expression also before
43   * deciding if a Cell can be considered deleted
44   */
45  @InterfaceAudience.Private
46  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
47  
48    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
49  
50    // Its better to track the visibility tags in delete based on each type.  Create individual
51    // data structures for tracking each of them.  This would ensure that there is no tracking based
52    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
53    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
54    // put will not be same. So going with individual data structures for different delete
55    // type would solve this problem and also ensure that the combination of different type
56    // of deletes with diff ts would also work fine
57    // Track per TS
58    private Map<Long, Pair<List<Tag>, Byte>> visibilityTagsDeleteFamily =
59        new HashMap<Long, Pair<List<Tag>, Byte>>();
60    // Delete family version with different ts and different visibility expression could come.
61    // Need to track it per ts.
62    private Map<Long,Pair<List<Tag>, Byte>> visibilityTagsDeleteFamilyVersion =
63        new HashMap<Long,Pair<List<Tag>, Byte>>();
64    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
65    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
66    // TODO : Need to handle puts with same ts but different vis tags.
67    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
68        new ArrayList<Pair<List<Tag>, Byte>>();
69  
70    public VisibilityScanDeleteTracker() {
71      super();
72    }
73  
74    @Override
75    public void add(Cell delCell) {
76      //Cannot call super.add because need to find if the delete needs to be considered
77      long timestamp = delCell.getTimestamp();
78      int qualifierOffset = delCell.getQualifierOffset();
79      int qualifierLength = delCell.getQualifierLength();
80      byte type = delCell.getTypeByte();
81      if (type == KeyValue.Type.DeleteFamily.getCode()) {
82        hasFamilyStamp = true;
83        //familyStamps.add(delCell.getTimestamp());
84        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
85        return;
86      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
87        familyVersionStamps.add(timestamp);
88        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
89        return;
90      }
91      // new column, or more general delete type
92      if (deleteBuffer != null) {
93        if (Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, delCell.getQualifierArray(),
94            qualifierOffset, qualifierLength) != 0) {
95          // A case where there are deletes for a column qualifier but there are
96          // no corresponding puts for them. Rare case.
97          visibilityTagsDeleteColumns = null;
98          visiblityTagsDeleteColumnVersion = null;
99        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
100         // there is a timestamp change which means we could clear the list
101         // when ts is same and the vis tags are different we need to collect
102         // them all. Interesting part is that in the normal case of puts if
103         // there are 2 cells with same ts and diff vis tags only one of them is
104         // returned. Handling with a single List<Tag> would mean that only one
105         // of the cell would be considered. Doing this as a precaution.
106         // Rare cases.
107         visiblityTagsDeleteColumnVersion = null;
108       }
109     }
110     deleteBuffer = delCell.getQualifierArray();
111     deleteOffset = qualifierOffset;
112     deleteLength = qualifierLength;
113     deleteType = type;
114     deleteTimestamp = timestamp;
115     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
116   }
117 
118   private void extractDeleteCellVisTags(Cell delCell, Type type) {
119     // If tag is present in the delete
120     if (delCell.getTagsLengthUnsigned() > 0) {
121       switch (type) {
122         case DeleteFamily:
123           List<Tag> delTags = new ArrayList<Tag>();
124           if (visibilityTagsDeleteFamily != null) {
125             Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
126             if (!delTags.isEmpty()) {
127               visibilityTagsDeleteFamily.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
128                   delTags, deleteCellVisTagsFormat));
129             }
130           }
131           break;
132         case DeleteFamilyVersion:
133           delTags = new ArrayList<Tag>();
134           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
135           if (!delTags.isEmpty()) {
136             visibilityTagsDeleteFamilyVersion.put(delCell.getTimestamp(), new Pair<List<Tag>, Byte>(
137                 delTags, deleteCellVisTagsFormat));
138           }
139           break;
140         case DeleteColumn:
141           if (visibilityTagsDeleteColumns == null) {
142             visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
143           }
144           delTags = new ArrayList<Tag>();
145           deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
146           if (!delTags.isEmpty()) {
147             visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
148                 deleteCellVisTagsFormat));
149           }
150           break;
151         case Delete:
152           if (visiblityTagsDeleteColumnVersion == null) {
153             visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
154           }
155           delTags = new ArrayList<Tag>();
156           deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
157           if (!delTags.isEmpty()) {
158             visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
159                 deleteCellVisTagsFormat));
160           }
161           break;
162         default:
163           throw new IllegalArgumentException("Invalid delete type");
164       }
165     } else {
166       switch (type) {
167         case DeleteFamily:
168           visibilityTagsDeleteFamily = null;
169           break;
170         case DeleteFamilyVersion:
171           visibilityTagsDeleteFamilyVersion = null;
172           break;
173         case DeleteColumn:
174           visibilityTagsDeleteColumns = null;
175           break;
176         case Delete:
177           visiblityTagsDeleteColumnVersion = null;
178           break;
179         default:
180           throw new IllegalArgumentException("Invalid delete type");
181       }
182     }
183   }
184 
185   @Override
186   public DeleteResult isDeleted(Cell cell) {
187     long timestamp = cell.getTimestamp();
188     int qualifierOffset = cell.getQualifierOffset();
189     int qualifierLength = cell.getQualifierLength();
190     try {
191       if (hasFamilyStamp) {
192         if (visibilityTagsDeleteFamily != null) {
193           Set<Entry<Long, Pair<List<Tag>, Byte>>> deleteFamilies = visibilityTagsDeleteFamily
194               .entrySet();
195           Iterator<Entry<Long, Pair<List<Tag>, Byte>>> iterator = deleteFamilies.iterator();
196           while (iterator.hasNext()) {
197             Entry<Long, Pair<List<Tag>, Byte>> entry = iterator.next();
198             if (timestamp <= entry.getKey()) {
199               List<Tag> putVisTags = new ArrayList<Tag>();
200               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
201               boolean matchFound = VisibilityLabelServiceManager
202                   .getInstance().getVisibilityLabelService()
203                   .matchVisibility(putVisTags, putCellVisTagsFormat, entry.getValue().getFirst(),
204                       entry.getValue().getSecond());
205               if (matchFound) {
206                 return DeleteResult.FAMILY_VERSION_DELETED;
207               }
208             }
209           }
210         } else {
211           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
212             // No tags
213             return DeleteResult.FAMILY_VERSION_DELETED;
214           }
215         }
216       }
217       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
218         if (visibilityTagsDeleteFamilyVersion != null) {
219           Pair<List<Tag>, Byte> tags = visibilityTagsDeleteFamilyVersion.get(Long
220               .valueOf(timestamp));
221           if (tags != null) {
222             List<Tag> putVisTags = new ArrayList<Tag>();
223             Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
224             boolean matchFound = VisibilityLabelServiceManager
225                 .getInstance()
226                 .getVisibilityLabelService()
227                 .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
228                     tags.getSecond());
229             if (matchFound) {
230               return DeleteResult.FAMILY_VERSION_DELETED;
231             }
232           }
233         } else {
234           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
235             // No tags
236             return DeleteResult.FAMILY_VERSION_DELETED;
237           }
238         }
239       }
240       if (deleteBuffer != null) {
241         int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength,
242             cell.getQualifierArray(), qualifierOffset, qualifierLength);
243 
244         if (ret == 0) {
245           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
246             if (visibilityTagsDeleteColumns != null) {
247               for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
248                 List<Tag> putVisTags = new ArrayList<Tag>();
249                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
250                 boolean matchFound = VisibilityLabelServiceManager
251                     .getInstance()
252                     .getVisibilityLabelService()
253                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
254                         tags.getSecond());
255                 if (matchFound) {
256                   return DeleteResult.VERSION_DELETED;
257                 }
258               }
259             } else {
260               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
261                 // No tags
262                 return DeleteResult.VERSION_DELETED;
263               }
264             }
265           }
266           // Delete (aka DeleteVersion)
267           // If the timestamp is the same, keep this one
268           if (timestamp == deleteTimestamp) {
269             if (visiblityTagsDeleteColumnVersion != null) {
270               for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
271                 List<Tag> putVisTags = new ArrayList<Tag>();
272                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
273                 boolean matchFound = VisibilityLabelServiceManager
274                     .getInstance()
275                     .getVisibilityLabelService()
276                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
277                         tags.getSecond());
278                 if (matchFound) {
279                   return DeleteResult.VERSION_DELETED;
280                 }
281               }
282             } else {
283               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
284                 // No tags
285                 return DeleteResult.VERSION_DELETED;
286               }
287             }
288           }
289         } else if (ret < 0) {
290           // Next column case.
291           deleteBuffer = null;
292           visibilityTagsDeleteColumns = null;
293           visiblityTagsDeleteColumnVersion = null;
294         } else {
295           throw new IllegalStateException("isDeleted failed: deleteBuffer="
296               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
297               + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength)
298               + ", timestamp=" + timestamp + ", comparison result: " + ret);
299         }
300       }
301     } catch (IOException e) {
302       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
303     }
304     return DeleteResult.NOT_DELETED;
305   }
306 
307   @Override
308   public void reset() {
309     super.reset();
310     visibilityTagsDeleteColumns = null;
311     visibilityTagsDeleteFamily = new HashMap<Long, Pair<List<Tag>, Byte>>();
312     visibilityTagsDeleteFamilyVersion = new HashMap<Long, Pair<List<Tag>, Byte>>();
313     visiblityTagsDeleteColumnVersion = null;
314   }
315 }