View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.thrift2;
19  
20  import static java.nio.ByteBuffer.wrap;
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.nio.ByteBuffer;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.ArrayList;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.List;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.testclassification.MediumTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.client.HBaseAdmin;
43  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
44  import org.apache.hadoop.hbase.security.User;
45  import org.apache.hadoop.hbase.security.UserProvider;
46  import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
47  import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
48  import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
49  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
50  import org.apache.hadoop.hbase.security.visibility.VisibilityController;
51  import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
52  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
53  import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
54  import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
55  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
56  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
57  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
58  import org.apache.hadoop.hbase.thrift2.generated.TGet;
59  import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
60  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
61  import org.apache.hadoop.hbase.thrift2.generated.TPut;
62  import org.apache.hadoop.hbase.thrift2.generated.TResult;
63  import org.apache.hadoop.hbase.thrift2.generated.TScan;
64  import org.apache.hadoop.hbase.util.Bytes;
65  import org.junit.AfterClass;
66  import org.junit.Assert;
67  import org.junit.Before;
68  import org.junit.BeforeClass;
69  import org.junit.Test;
70  import org.junit.experimental.categories.Category;
71  
72  @Category(MediumTests.class)
73  public class TestThriftHBaseServiceHandlerWithLabels {
74  
75  public static final Log LOG = LogFactory
76      .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
77  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
78  
79  // Static names for tables, columns, rows, and values
80  private static byte[] tableAname = Bytes.toBytes("tableA");
81  private static byte[] familyAname = Bytes.toBytes("familyA");
82  private static byte[] familyBname = Bytes.toBytes("familyB");
83  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
84  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
85  private static byte[] valueAname = Bytes.toBytes("valueA");
86  private static byte[] valueBname = Bytes.toBytes("valueB");
87  private static HColumnDescriptor[] families = new HColumnDescriptor[] {
88      new HColumnDescriptor(familyAname).setMaxVersions(3),
89      new HColumnDescriptor(familyBname).setMaxVersions(2) };
90  
91  private final static String TOPSECRET = "topsecret";
92  private final static String PUBLIC = "public";
93  private final static String PRIVATE = "private";
94  private final static String CONFIDENTIAL = "confidential";
95  private final static String SECRET = "secret";
96  private static User SUPERUSER;
97  
98  private static Configuration conf;
99  
100 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
101     List<TColumnValue> columnValuesB) {
102   assertEquals(columnValuesA.size(), columnValuesB.size());
103   Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
104     @Override
105     public int compare(TColumnValue o1, TColumnValue o2) {
106       return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
107           Bytes.add(o2.getFamily(), o2.getQualifier()));
108     }
109   };
110   Collections.sort(columnValuesA, comparator);
111   Collections.sort(columnValuesB, comparator);
112 
113   for (int i = 0; i < columnValuesA.size(); i++) {
114     TColumnValue a = columnValuesA.get(i);
115     TColumnValue b = columnValuesB.get(i);
116     assertArrayEquals(a.getFamily(), b.getFamily());
117     assertArrayEquals(a.getQualifier(), b.getQualifier());
118     assertArrayEquals(a.getValue(), b.getValue());
119   }
120 }
121 
122 @BeforeClass
123 public static void beforeClass() throws Exception {
124   SUPERUSER = User.createUserForTesting(conf, "admin",
125       new String[] { "supergroup" });
126   conf = UTIL.getConfiguration();
127   conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
128       SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
129   conf.set("hbase.superuser", SUPERUSER.getShortName());
130   conf.set("hbase.coprocessor.master.classes",
131       VisibilityController.class.getName());
132   conf.set("hbase.coprocessor.region.classes",
133       VisibilityController.class.getName());
134   conf.setInt("hfile.format.version", 3);
135   UTIL.startMiniCluster(1);
136   // Wait for the labels table to become available
137   UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
138   createLabels();
139   HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
140   HTableDescriptor tableDescriptor = new HTableDescriptor(
141       TableName.valueOf(tableAname));
142   for (HColumnDescriptor family : families) {
143     tableDescriptor.addFamily(family);
144   }
145   admin.createTable(tableDescriptor);
146   admin.close();
147   setAuths();
148 }
149 
150 private static void createLabels() throws IOException, InterruptedException {
151   PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
152     public VisibilityLabelsResponse run() throws Exception {
153       String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
154       try {
155         VisibilityClient.addLabels(conf, labels);
156       } catch (Throwable t) {
157         throw new IOException(t);
158       }
159       return null;
160     }
161   };
162   SUPERUSER.runAs(action);
163 }
164 
165 private static void setAuths() throws IOException {
166   String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
167   try {
168     VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
169   } catch (Throwable t) {
170     throw new IOException(t);
171   }
172 }
173 
174 @AfterClass
175 public static void afterClass() throws Exception {
176   UTIL.shutdownMiniCluster();
177 }
178 
179 @Before
180 public void setup() throws Exception {
181 
182 }
183 
184 private ThriftHBaseServiceHandler createHandler() throws IOException {
185   return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
186 }
187 
188 @Test
189 public void testScanWithVisibilityLabels() throws Exception {
190   ThriftHBaseServiceHandler handler = createHandler();
191   ByteBuffer table = wrap(tableAname);
192 
193   // insert data
194   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
195       wrap(qualifierAname), wrap(valueAname));
196   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
197   columnValues.add(columnValue);
198   for (int i = 0; i < 10; i++) {
199     TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
200     if (i == 5) {
201       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
202     } else {
203       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
204           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
205     }
206     handler.put(table, put);
207   }
208 
209   // create scan instance
210   TScan scan = new TScan();
211   List<TColumn> columns = new ArrayList<TColumn>();
212   TColumn column = new TColumn();
213   column.setFamily(familyAname);
214   column.setQualifier(qualifierAname);
215   columns.add(column);
216   scan.setColumns(columns);
217   scan.setStartRow("testScan".getBytes());
218   scan.setStopRow("testScan\uffff".getBytes());
219 
220   TAuthorization tauth = new TAuthorization();
221   List<String> labels = new ArrayList<String>();
222   labels.add(SECRET);
223   labels.add(PRIVATE);
224   tauth.setLabels(labels);
225   scan.setAuthorizations(tauth);
226   // get scanner and rows
227   int scanId = handler.openScanner(table, scan);
228   List<TResult> results = handler.getScannerRows(scanId, 10);
229   assertEquals(9, results.size());
230   Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
231       ("testScan" + 5).getBytes()));
232   for (int i = 0; i < 9; i++) {
233     if (i < 5) {
234       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
235     } else if (i == 5) {
236       continue;
237     } else {
238       assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
239           .getRow());
240     }
241   }
242 
243   // check that we are at the end of the scan
244   results = handler.getScannerRows(scanId, 9);
245   assertEquals(0, results.size());
246 
247   // close scanner and check that it was indeed closed
248   handler.closeScanner(scanId);
249   try {
250     handler.getScannerRows(scanId, 9);
251     fail("Scanner id should be invalid");
252   } catch (TIllegalArgument e) {
253   }
254 }
255 
256 @Test
257 public void testGetScannerResultsWithAuthorizations() throws Exception {
258   ThriftHBaseServiceHandler handler = createHandler();
259   ByteBuffer table = wrap(tableAname);
260 
261   // insert data
262   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
263       wrap(qualifierAname), wrap(valueAname));
264   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
265   columnValues.add(columnValue);
266   for (int i = 0; i < 20; i++) {
267     TPut put = new TPut(
268         wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
269         columnValues);
270     if (i == 3) {
271       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
272     } else {
273       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
274           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
275     }
276     handler.put(table, put);
277   }
278 
279   // create scan instance
280   TScan scan = new TScan();
281   List<TColumn> columns = new ArrayList<TColumn>();
282   TColumn column = new TColumn();
283   column.setFamily(familyAname);
284   column.setQualifier(qualifierAname);
285   columns.add(column);
286   scan.setColumns(columns);
287   scan.setStartRow("testGetScannerResults".getBytes());
288 
289   // get 5 rows and check the returned results
290   scan.setStopRow("testGetScannerResults05".getBytes());
291   TAuthorization tauth = new TAuthorization();
292   List<String> labels = new ArrayList<String>();
293   labels.add(SECRET);
294   labels.add(PRIVATE);
295   tauth.setLabels(labels);
296   scan.setAuthorizations(tauth);
297   List<TResult> results = handler.getScannerResults(table, scan, 5);
298   assertEquals(4, results.size());
299   for (int i = 0; i < 4; i++) {
300     if (i < 3) {
301       assertArrayEquals(
302           ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
303           results.get(i).getRow());
304     } else if (i == 3) {
305       continue;
306     } else {
307       assertArrayEquals(
308           ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
309               .get(i).getRow());
310     }
311   }
312 }
313 
314 @Test
315 public void testGetsWithLabels() throws Exception {
316   ThriftHBaseServiceHandler handler = createHandler();
317   byte[] rowName = "testPutGet".getBytes();
318   ByteBuffer table = wrap(tableAname);
319 
320   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
321   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
322       wrap(valueAname)));
323   columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
324       wrap(valueBname)));
325   TPut put = new TPut(wrap(rowName), columnValues);
326 
327   put.setColumnValues(columnValues);
328   put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
329       + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
330   handler.put(table, put);
331   TGet get = new TGet(wrap(rowName));
332   TAuthorization tauth = new TAuthorization();
333   List<String> labels = new ArrayList<String>();
334   labels.add(SECRET);
335   labels.add(PRIVATE);
336   tauth.setLabels(labels);
337   get.setAuthorizations(tauth);
338   TResult result = handler.get(table, get);
339   assertArrayEquals(rowName, result.getRow());
340   List<TColumnValue> returnedColumnValues = result.getColumnValues();
341   assertTColumnValuesEqual(columnValues, returnedColumnValues);
342 }
343 
344 @Test
345 public void testIncrementWithTags() throws Exception {
346   ThriftHBaseServiceHandler handler = createHandler();
347   byte[] rowName = "testIncrementWithTags".getBytes();
348   ByteBuffer table = wrap(tableAname);
349 
350   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
351   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
352       wrap(Bytes.toBytes(1L))));
353   TPut put = new TPut(wrap(rowName), columnValues);
354   put.setColumnValues(columnValues);
355   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
356   handler.put(table, put);
357 
358   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
359   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
360       wrap(qualifierAname)));
361   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
362   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
363   handler.increment(table, increment);
364 
365   TGet get = new TGet(wrap(rowName));
366   TAuthorization tauth = new TAuthorization();
367   List<String> labels = new ArrayList<String>();
368   labels.add(SECRET);
369   tauth.setLabels(labels);
370   get.setAuthorizations(tauth);
371   TResult result = handler.get(table, get);
372 
373   assertArrayEquals(rowName, result.getRow());
374   assertEquals(1, result.getColumnValuesSize());
375   TColumnValue columnValue = result.getColumnValues().get(0);
376   assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
377 }
378 
379 @Test
380 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
381   ThriftHBaseServiceHandler handler = createHandler();
382   byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
383   ByteBuffer table = wrap(tableAname);
384 
385   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
386   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
387       wrap(Bytes.toBytes(1L))));
388   TPut put = new TPut(wrap(rowName), columnValues);
389   put.setColumnValues(columnValues);
390   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
391   handler.put(table, put);
392 
393   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
394   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
395       wrap(qualifierAname)));
396   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
397   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
398   handler.increment(table, increment);
399 
400   TGet get = new TGet(wrap(rowName));
401   TAuthorization tauth = new TAuthorization();
402   List<String> labels = new ArrayList<String>();
403   labels.add(PUBLIC);
404   tauth.setLabels(labels);
405   get.setAuthorizations(tauth);
406   TResult result = handler.get(table, get);
407   assertNull(result.getRow());
408 }
409 
410 @Test
411 public void testAppend() throws Exception {
412   ThriftHBaseServiceHandler handler = createHandler();
413   byte[] rowName = "testAppend".getBytes();
414   ByteBuffer table = wrap(tableAname);
415   byte[] v1 = Bytes.toBytes(1L);
416   byte[] v2 = Bytes.toBytes(5L);
417   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
418   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
419       wrap(Bytes.toBytes(1L))));
420   TPut put = new TPut(wrap(rowName), columnValues);
421   put.setColumnValues(columnValues);
422   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
423   handler.put(table, put);
424 
425   List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
426   appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
427       wrap(v2)));
428   TAppend append = new TAppend(wrap(rowName), appendColumns);
429   append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
430   handler.append(table, append);
431 
432   TGet get = new TGet(wrap(rowName));
433   TAuthorization tauth = new TAuthorization();
434   List<String> labels = new ArrayList<String>();
435   labels.add(SECRET);
436   tauth.setLabels(labels);
437   get.setAuthorizations(tauth);
438   TResult result = handler.get(table, get);
439 
440   assertArrayEquals(rowName, result.getRow());
441   assertEquals(1, result.getColumnValuesSize());
442   TColumnValue columnValue = result.getColumnValues().get(0);
443   assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
444 }
445 
446 /**
447  * Padding numbers to make comparison of sort order easier in a for loop
448  * 
449  * @param n
450  *          The number to pad.
451  * @param pad
452  *          The length to pad up to.
453  * @return The padded number as a string.
454  */
455 private String pad(int n, byte pad) {
456   String res = Integer.toString(n);
457   while (res.length() < pad)
458     res = "0" + res;
459   return res;
460 }
461 }