1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.IOException;
22 import java.util.Iterator;
23 import java.util.List;
24 import java.util.NavigableMap;
25
26 import junit.framework.AssertionFailedError;
27 import junit.framework.TestCase;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.client.Delete;
35 import org.apache.hadoop.hbase.client.Durability;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.HTable;
38 import org.apache.hadoop.hbase.client.Put;
39 import org.apache.hadoop.hbase.client.Result;
40 import org.apache.hadoop.hbase.client.ResultScanner;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.regionserver.HRegion;
43 import org.apache.hadoop.hbase.regionserver.InternalScanner;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.FSTableDescriptors;
46 import org.apache.hadoop.hbase.util.FSUtils;
47 import org.apache.hadoop.hdfs.MiniDFSCluster;
48
49
50
51
52
53
54 public abstract class HBaseTestCase extends TestCase {
55 private static final Log LOG = LogFactory.getLog(HBaseTestCase.class);
56
57 protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
58 protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
59 protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
60
61 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
62
63 private boolean localfs = false;
64 protected static Path testDir = null;
65 protected FileSystem fs = null;
66 protected HRegion meta = null;
67 protected static final char FIRST_CHAR = 'a';
68 protected static final char LAST_CHAR = 'z';
69 protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
70 protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
71 protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
72 protected static final int MAXVERSIONS = 3;
73
74 protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
75
76 public volatile Configuration conf = HBaseConfiguration.create();
77 public final FSTableDescriptors fsTableDescriptors;
78 {
79 try {
80 fsTableDescriptors = new FSTableDescriptors(conf);
81 } catch (IOException e) {
82 throw new RuntimeException("Failed to init descriptors", e);
83 }
84 }
85
86
87 public HBaseTestCase() {
88 super();
89 }
90
91
92
93
94 public HBaseTestCase(String name) {
95 super(name);
96 }
97
98
99
100
101
102 @Override
103 protected void setUp() throws Exception {
104 super.setUp();
105 localfs =
106 (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
107
108 if (fs == null) {
109 this.fs = FileSystem.get(conf);
110 }
111 try {
112 if (localfs) {
113 this.testDir = getUnitTestdir(getName());
114 if (fs.exists(testDir)) {
115 fs.delete(testDir, true);
116 }
117 } else {
118 this.testDir = FSUtils.getRootDir(conf);
119 }
120 } catch (Exception e) {
121 LOG.fatal("error during setup", e);
122 throw e;
123 }
124 }
125
126 @Override
127 protected void tearDown() throws Exception {
128 try {
129 if (localfs) {
130 if (this.fs.exists(testDir)) {
131 this.fs.delete(testDir, true);
132 }
133 }
134 } catch (Exception e) {
135 LOG.fatal("error during tear down", e);
136 }
137 super.tearDown();
138 }
139
140
141
142
143
144
145 protected Path getUnitTestdir(String testName) {
146 return testUtil.getDataTestDir(testName);
147 }
148
149
150
151
152
153
154
155
156
157
158
159 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
160 byte [] endKey)
161 throws IOException {
162 return createNewHRegion(desc, startKey, endKey, this.conf);
163 }
164
165 public HRegion createNewHRegion(HTableDescriptor desc, byte [] startKey,
166 byte [] endKey, Configuration conf)
167 throws IOException {
168 HRegionInfo hri = new HRegionInfo(desc.getTableName(), startKey, endKey);
169 return HRegion.createHRegion(hri, testDir, conf, desc);
170 }
171
172 protected HRegion openClosedRegion(final HRegion closedRegion)
173 throws IOException {
174 return HRegion.openHRegion(closedRegion, null);
175 }
176
177
178
179
180
181
182
183 protected HTableDescriptor createTableDescriptor(final String name) {
184 return createTableDescriptor(name, MAXVERSIONS);
185 }
186
187
188
189
190
191
192
193
194 protected HTableDescriptor createTableDescriptor(final String name,
195 final int versions) {
196 return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
197 versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
198 }
199
200
201
202
203
204
205
206
207 protected HTableDescriptor createTableDescriptor(final String name,
208 final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
209 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
210 for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
211 htd.addFamily(new HColumnDescriptor(cfName)
212 .setMinVersions(minVersions)
213 .setMaxVersions(versions)
214 .setKeepDeletedCells(keepDeleted)
215 .setBlockCacheEnabled(false)
216 .setTimeToLive(ttl)
217 );
218 }
219 return htd;
220 }
221
222
223
224
225
226
227
228
229
230
231
232 public static long addContent(final HRegion r, final byte [] columnFamily, final byte[] column)
233 throws IOException {
234 byte [] startKey = r.getRegionInfo().getStartKey();
235 byte [] endKey = r.getRegionInfo().getEndKey();
236 byte [] startKeyBytes = startKey;
237 if (startKeyBytes == null || startKeyBytes.length == 0) {
238 startKeyBytes = START_KEY_BYTES;
239 }
240 return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), Bytes.toString(column),
241 startKeyBytes, endKey, -1);
242 }
243
244 public static long addContent(final HRegion r, final byte [] columnFamily)
245 throws IOException {
246 return addContent(r, columnFamily, null);
247 }
248
249
250
251
252
253
254
255
256
257
258
259 public static long addContent(final Incommon updater,
260 final String columnFamily) throws IOException {
261 return addContent(updater, columnFamily, START_KEY_BYTES, null);
262 }
263
264 public static long addContent(final Incommon updater, final String family,
265 final String column) throws IOException {
266 return addContent(updater, family, column, START_KEY_BYTES, null);
267 }
268
269
270
271
272
273
274
275
276
277
278
279
280
281 public static long addContent(final Incommon updater, final String columnFamily,
282 final byte [] startKeyBytes, final byte [] endKey)
283 throws IOException {
284 return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
285 }
286
287 public static long addContent(final Incommon updater, final String family, String column,
288 final byte [] startKeyBytes, final byte [] endKey) throws IOException {
289 return addContent(updater, family, column, startKeyBytes, endKey, -1);
290 }
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305 public static long addContent(final Incommon updater,
306 final String columnFamily,
307 final String column,
308 final byte [] startKeyBytes, final byte [] endKey, final long ts)
309 throws IOException {
310 long count = 0;
311
312
313
314
315 char secondCharStart = (char)startKeyBytes[1];
316 char thirdCharStart = (char)startKeyBytes[2];
317 EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
318 for (char d = secondCharStart; d <= LAST_CHAR; d++) {
319 for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
320 byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
321 if (endKey != null && endKey.length > 0
322 && Bytes.compareTo(endKey, t) <= 0) {
323 break EXIT;
324 }
325 try {
326 Put put;
327 if(ts != -1) {
328 put = new Put(t, ts);
329 } else {
330 put = new Put(t);
331 }
332 try {
333 StringBuilder sb = new StringBuilder();
334 if (column != null && column.contains(":")) {
335 sb.append(column);
336 } else {
337 if (columnFamily != null) {
338 sb.append(columnFamily);
339 if (!columnFamily.endsWith(":")) {
340 sb.append(":");
341 }
342 if (column != null) {
343 sb.append(column);
344 }
345 }
346 }
347 byte[][] split =
348 KeyValue.parseColumn(Bytes.toBytes(sb.toString()));
349 if(split.length == 1) {
350 put.add(split[0], new byte[0], t);
351 } else {
352 put.add(split[0], split[1], t);
353 }
354 put.setDurability(Durability.SKIP_WAL);
355 updater.put(put);
356 count++;
357 } catch (RuntimeException ex) {
358 ex.printStackTrace();
359 throw ex;
360 } catch (IOException ex) {
361 ex.printStackTrace();
362 throw ex;
363 }
364 } catch (RuntimeException ex) {
365 ex.printStackTrace();
366 throw ex;
367 } catch (IOException ex) {
368 ex.printStackTrace();
369 throw ex;
370 }
371 }
372
373 thirdCharStart = FIRST_CHAR;
374 }
375 secondCharStart = FIRST_CHAR;
376 }
377 return count;
378 }
379
380
381
382
383 public interface FlushCache {
384
385
386
387 void flushcache() throws IOException;
388 }
389
390
391
392
393
394
395
396 public interface Incommon {
397
398
399
400
401
402
403 void delete(Delete delete, boolean writeToWAL)
404 throws IOException;
405
406
407
408
409
410 void put(Put put) throws IOException;
411
412 Result get(Get get) throws IOException;
413
414
415
416
417
418
419
420
421
422 ScannerIncommon getScanner(
423 byte[] family, byte[][] qualifiers, byte[] firstRow, long ts
424 )
425 throws IOException;
426 }
427
428
429
430
431 public static class HRegionIncommon implements Incommon, FlushCache {
432 final HRegion region;
433
434
435
436
437 public HRegionIncommon(final HRegion HRegion) {
438 this.region = HRegion;
439 }
440
441 public void put(Put put) throws IOException {
442 region.put(put);
443 }
444
445 public void delete(Delete delete, boolean writeToWAL)
446 throws IOException {
447 this.region.delete(delete);
448 }
449
450 public Result get(Get get) throws IOException {
451 return region.get(get);
452 }
453
454 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
455 byte [] firstRow, long ts)
456 throws IOException {
457 Scan scan = new Scan(firstRow);
458 if(qualifiers == null || qualifiers.length == 0) {
459 scan.addFamily(family);
460 } else {
461 for(int i=0; i<qualifiers.length; i++){
462 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
463 }
464 }
465 scan.setTimeRange(0, ts);
466 return new
467 InternalScannerIncommon(region.getScanner(scan));
468 }
469
470 public void flushcache() throws IOException {
471 this.region.flushcache();
472 }
473 }
474
475
476
477
478 public static class HTableIncommon implements Incommon {
479 final HTable table;
480
481
482
483
484 public HTableIncommon(final HTable table) {
485 super();
486 this.table = table;
487 }
488
489 public void put(Put put) throws IOException {
490 table.put(put);
491 }
492
493
494 public void delete(Delete delete, boolean writeToWAL)
495 throws IOException {
496 this.table.delete(delete);
497 }
498
499 public Result get(Get get) throws IOException {
500 return table.get(get);
501 }
502
503 public ScannerIncommon getScanner(byte [] family, byte [][] qualifiers,
504 byte [] firstRow, long ts)
505 throws IOException {
506 Scan scan = new Scan(firstRow);
507 if(qualifiers == null || qualifiers.length == 0) {
508 scan.addFamily(family);
509 } else {
510 for(int i=0; i<qualifiers.length; i++){
511 scan.addColumn(HConstants.CATALOG_FAMILY, qualifiers[i]);
512 }
513 }
514 scan.setTimeRange(0, ts);
515 return new
516 ClientScannerIncommon(table.getScanner(scan));
517 }
518 }
519
520 public interface ScannerIncommon
521 extends Iterable<Result> {
522 boolean next(List<Cell> values)
523 throws IOException;
524
525 void close() throws IOException;
526 }
527
528 public static class ClientScannerIncommon implements ScannerIncommon {
529 ResultScanner scanner;
530 public ClientScannerIncommon(ResultScanner scanner) {
531 this.scanner = scanner;
532 }
533
534 @Override
535 public boolean next(List<Cell> values)
536 throws IOException {
537 Result results = scanner.next();
538 if (results == null) {
539 return false;
540 }
541 values.clear();
542 values.addAll(results.listCells());
543 return true;
544 }
545
546 public void close() throws IOException {
547 scanner.close();
548 }
549
550 public Iterator<Result> iterator() {
551 return scanner.iterator();
552 }
553 }
554
555 public static class InternalScannerIncommon implements ScannerIncommon {
556 InternalScanner scanner;
557
558 public InternalScannerIncommon(InternalScanner scanner) {
559 this.scanner = scanner;
560 }
561
562 @Override
563 public boolean next(List<Cell> results)
564 throws IOException {
565 return scanner.next(results);
566 }
567
568 @Override
569 public void close() throws IOException {
570 scanner.close();
571 }
572
573 @Override
574 public Iterator<Result> iterator() {
575 throw new UnsupportedOperationException();
576 }
577 }
578
579 protected void assertResultEquals(final HRegion region, final byte [] row,
580 final byte [] family, final byte [] qualifier, final long timestamp,
581 final byte [] value)
582 throws IOException {
583 Get get = new Get(row);
584 get.setTimeStamp(timestamp);
585 Result res = region.get(get);
586 NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
587 res.getMap();
588 byte [] res_value = map.get(family).get(qualifier).get(timestamp);
589
590 if (value == null) {
591 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
592 " at timestamp " + timestamp, null, res_value);
593 } else {
594 if (res_value == null) {
595 fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
596 " at timestamp " + timestamp + "\" was expected to be \"" +
597 Bytes.toStringBinary(value) + " but was null");
598 }
599 if (res_value != null) {
600 assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
601 " at timestamp " +
602 timestamp, value, new String(res_value));
603 }
604 }
605 }
606
607
608
609
610
611
612 public static void shutdownDfs(MiniDFSCluster cluster) {
613 if (cluster != null) {
614 LOG.info("Shutting down Mini DFS ");
615 try {
616 cluster.shutdown();
617 } catch (Exception e) {
618
619
620
621 }
622 try {
623 FileSystem fs = cluster.getFileSystem();
624 if (fs != null) {
625 LOG.info("Shutting down FileSystem");
626 fs.close();
627 }
628 FileSystem.closeAll();
629 } catch (IOException e) {
630 LOG.error("error closing file system", e);
631 }
632 }
633 }
634
635
636
637
638
639
640 protected void createMetaRegion() throws IOException {
641 FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
642 meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf,
643 fsTableDescriptors.get(TableName.META_TABLE_NAME));
644 }
645
646 protected void closeRootAndMeta() throws IOException {
647 HRegion.closeHRegion(meta);
648 }
649
650 public static void assertByteEquals(byte[] expected,
651 byte[] actual) {
652 if (Bytes.compareTo(expected, actual) != 0) {
653 throw new AssertionFailedError("expected:<" +
654 Bytes.toString(expected) + "> but was:<" +
655 Bytes.toString(actual) + ">");
656 }
657 }
658
659 public static void assertEquals(byte[] expected,
660 byte[] actual) {
661 if (Bytes.compareTo(expected, actual) != 0) {
662 throw new AssertionFailedError("expected:<" +
663 Bytes.toStringBinary(expected) + "> but was:<" +
664 Bytes.toStringBinary(actual) + ">");
665 }
666 }
667
668 }