View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.filter;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Arrays;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.*;
30  import org.apache.hadoop.hbase.client.Put;
31  import org.apache.hadoop.hbase.client.Scan;
32  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
33  import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
34  import org.apache.hadoop.hbase.regionserver.HRegion;
35  import org.apache.hadoop.hbase.regionserver.InternalScanner;
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.apache.hadoop.hbase.util.Bytes;
38  
39  import org.junit.After;
40  import org.junit.Before;
41  import org.junit.Test;
42  import static org.junit.Assert.assertEquals;
43  import static org.junit.Assert.assertNotNull;
44  import static org.junit.Assert.assertTrue;
45  
46  import org.junit.experimental.categories.Category;
47  
48  @Category(SmallTests.class)
49  public class TestDependentColumnFilter {
50    private final Log LOG = LogFactory.getLog(this.getClass());
51    private static final byte[][] ROWS = {
52  	  Bytes.toBytes("test1"),Bytes.toBytes("test2")
53    };
54    private static final byte[][] FAMILIES = {
55  	  Bytes.toBytes("familyOne"),Bytes.toBytes("familyTwo")
56    };
57    private static final long STAMP_BASE = System.currentTimeMillis();
58    private static final long[] STAMPS = {
59  	  STAMP_BASE-100, STAMP_BASE-200, STAMP_BASE-300
60    };
61    private static final byte[] QUALIFIER = Bytes.toBytes("qualifier");
62    private static final byte[][] BAD_VALS = {
63  	  Bytes.toBytes("bad1"), Bytes.toBytes("bad2"), Bytes.toBytes("bad3")
64    };
65    private static final byte[] MATCH_VAL = Bytes.toBytes("match");
66    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67  
68    List<KeyValue> testVals;
69    private HRegion region;
70  
71    @Before
72    public void setUp() throws Exception {
73      testVals = makeTestVals();
74  
75      HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(this.getClass().getSimpleName()));
76      HColumnDescriptor hcd0 = new HColumnDescriptor(FAMILIES[0]);
77      hcd0.setMaxVersions(3);
78      htd.addFamily(hcd0);
79      HColumnDescriptor hcd1 = new HColumnDescriptor(FAMILIES[1]);
80      hcd1.setMaxVersions(3);
81      htd.addFamily(hcd1);
82      HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
83      this.region = HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(),
84        TEST_UTIL.getConfiguration(), htd);
85      addData();
86    }
87  
88    @After
89    public void tearDown() throws Exception {
90      HRegion.closeHRegion(this.region);
91    }
92  
93    private void addData() throws IOException {
94      Put put = new Put(ROWS[0]);
95      // add in an entry for each stamp, with 2 as a "good" value
96      put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
97      put.add(FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]);
98      put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
99      // add in entries for stamps 0 and 2.
100     // without a value check both will be "accepted"
101     // with one 2 will be accepted(since the corresponding ts entry
102     // has a matching value
103     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], BAD_VALS[0]);
104     put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
105 
106     this.region.put(put);
107 
108     put = new Put(ROWS[1]);
109     put.add(FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]);
110     // there is no corresponding timestamp for this so it should never pass
111     put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
112     // if we reverse the qualifiers this one should pass
113     put.add(FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL);
114     // should pass
115     put.add(FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]);
116 
117     this.region.put(put);
118   }
119 
120   private List<KeyValue> makeTestVals() {
121 	List<KeyValue> testVals = new ArrayList<KeyValue>();
122 	testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[0], BAD_VALS[0]));
123 	testVals.add(new KeyValue(ROWS[0], FAMILIES[0], QUALIFIER, STAMPS[1], BAD_VALS[1]));
124 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[1], BAD_VALS[2]));
125 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[0], MATCH_VAL));
126 	testVals.add(new KeyValue(ROWS[0], FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]));
127 
128 	return testVals;
129   }
130 
131   /**
132    * This shouldn't be confused with TestFilter#verifyScan
133    * as expectedKeys is not the per row total, but the scan total
134    *
135    * @param s
136    * @param expectedRows
137    * @param expectedCells
138    * @throws IOException
139    */
140   private void verifyScan(Scan s, long expectedRows, long expectedCells)
141   throws IOException {
142     InternalScanner scanner = this.region.getScanner(s);
143     List<Cell> results = new ArrayList<Cell>();
144     int i = 0;
145     int cells = 0;
146     for (boolean done = true; done; i++) {
147       done = scanner.next(results);
148       Arrays.sort(results.toArray(new KeyValue[results.size()]),
149           KeyValue.COMPARATOR);
150       LOG.info("counter=" + i + ", " + results);
151       if (results.isEmpty()) break;
152       cells += results.size();
153       assertTrue("Scanned too many rows! Only expected " + expectedRows +
154           " total but already scanned " + (i+1), expectedRows > i);
155       assertTrue("Expected " + expectedCells + " cells total but " +
156           "already scanned " + cells, expectedCells >= cells);
157       results.clear();
158     }
159     assertEquals("Expected " + expectedRows + " rows but scanned " + i +
160         " rows", expectedRows, i);
161     assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
162             " cells", expectedCells, cells);
163   }
164 
165   /**
166    * Test scans using a DependentColumnFilter
167    */
168   @Test
169   public void testScans() throws Exception {
170     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
171 
172     Scan scan = new Scan();
173     scan.setFilter(filter);
174     scan.setMaxVersions(Integer.MAX_VALUE);
175 
176     verifyScan(scan, 2, 8);
177 
178     // drop the filtering cells
179     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true);
180     scan = new Scan();
181     scan.setFilter(filter);
182     scan.setMaxVersions(Integer.MAX_VALUE);
183 
184     verifyScan(scan, 2, 3);
185 
186     // include a comparator operation
187     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, false,
188         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
189     scan = new Scan();
190     scan.setFilter(filter);
191     scan.setMaxVersions(Integer.MAX_VALUE);
192 
193     /*
194      * expecting to get the following 3 cells
195      * row 0
196      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
197      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
198      * row 1
199      *   put.add(FAMILIES[0], QUALIFIER, STAMPS[2], MATCH_VAL);
200      */
201     verifyScan(scan, 2, 3);
202 
203     // include a comparator operation and drop comparator
204     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true,
205         CompareOp.EQUAL, new BinaryComparator(MATCH_VAL));
206     scan = new Scan();
207     scan.setFilter(filter);
208     scan.setMaxVersions(Integer.MAX_VALUE);
209 
210     /*
211      * expecting to get the following 1 cell
212      * row 0
213      *   put.add(FAMILIES[1], QUALIFIER, STAMPS[2], BAD_VALS[2]);
214      */
215     verifyScan(scan, 1, 1);
216 
217   }
218 
219   /**
220    * Test that the filter correctly drops rows without a corresponding timestamp
221    *
222    * @throws Exception
223    */
224   @Test
225   public void testFilterDropping() throws Exception {
226     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
227     List<Cell> accepted = new ArrayList<Cell>();
228     for(Cell val : testVals) {
229       if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
230         accepted.add(val);
231       }
232     }
233     assertEquals("check all values accepted from filterKeyValue", 5, accepted.size());
234 
235     filter.filterRowCells(accepted);
236     assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size());
237 
238     // start do it again with dependent column dropping on
239     filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true);
240     accepted.clear();
241     for(KeyValue val : testVals) {
242         if(filter.filterKeyValue(val) == ReturnCode.INCLUDE) {
243           accepted.add(val);
244         }
245       }
246       assertEquals("check the filtering column cells got dropped", 2, accepted.size());
247 
248       filter.filterRowCells(accepted);
249       assertEquals("check cell retention", 2, accepted.size());
250   }
251 
252   /**
253    * Test for HBASE-8794. Avoid NullPointerException in DependentColumnFilter.toString().
254    */
255   @Test
256   public void testToStringWithNullComparator() {
257     // Test constructor that implicitly sets a null comparator
258     Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER);
259     assertNotNull(filter.toString());
260     assertTrue("check string contains 'null' as compatator is null",
261       filter.toString().contains("null"));
262 
263     // Test constructor with explicit null comparator
264     filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL, null);
265     assertNotNull(filter.toString());
266     assertTrue("check string contains 'null' as compatator is null",
267       filter.toString().contains("null"));
268   }
269 
270   @Test
271   public void testToStringWithNonNullComparator() {
272     Filter filter =
273         new DependentColumnFilter(FAMILIES[0], QUALIFIER, true, CompareOp.EQUAL,
274             new BinaryComparator(MATCH_VAL));
275     assertNotNull(filter.toString());
276     assertTrue("check string contains comparator value", filter.toString().contains("match"));
277   }
278 
279 }
280