1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.*;
26 import org.apache.hadoop.hbase.client.HBaseAdmin;
27 import org.apache.hadoop.hbase.regionserver.HRegion;
28 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29 import org.apache.hadoop.hbase.testclassification.MediumTests;
30 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32 import org.apache.hadoop.hdfs.MiniDFSCluster;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.ServerLoad;
36 import org.apache.hadoop.hbase.RegionLoad;
37
38 import java.io.*;
39 import java.util.*;
40
41 import org.junit.*;
42 import org.junit.experimental.categories.Category;
43
44 import static org.junit.Assert.assertEquals;
45 import static org.junit.Assert.assertNotNull;
46 import static org.junit.Assert.assertTrue;
47 import static org.junit.Assert.assertFalse;
48
49
50
51
52 @Category(MediumTests.class)
53 public class TestClassLoading {
54 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
55 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 private static MiniDFSCluster cluster;
58
59 static final String tableName = "TestClassLoading";
60 static final String cpName1 = "TestCP1";
61 static final String cpName2 = "TestCP2";
62 static final String cpName3 = "TestCP3";
63 static final String cpName4 = "TestCP4";
64 static final String cpName5 = "TestCP5";
65 static final String cpName6 = "TestCP6";
66
67 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68
69 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72
73 private static final String[] regionServerSystemCoprocessors =
74 new String[]{
75 regionServerCoprocessor.getSimpleName()
76 };
77
78 @BeforeClass
79 public static void setUpBeforeClass() throws Exception {
80 Configuration conf = TEST_UTIL.getConfiguration();
81
82
83
84 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
85 regionCoprocessor1.getName());
86
87
88
89
90 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
91 regionCoprocessor2.getName());
92
93 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
94 regionServerCoprocessor.getName());
95 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
96 masterCoprocessor.getName());
97 TEST_UTIL.startMiniCluster(1);
98 cluster = TEST_UTIL.getDFSCluster();
99 }
100
101 @AfterClass
102 public static void tearDownAfterClass() throws Exception {
103 TEST_UTIL.shutdownMiniCluster();
104 }
105
106 static File buildCoprocessorJar(String className) throws Exception {
107 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
108 "public class " + className + " extends BaseRegionObserver {}";
109 return ClassLoaderTestHelper.buildJar(
110 TEST_UTIL.getDataTestDir().toString(), className, code);
111 }
112
113 @Test
114
115 public void testClassLoadingFromHDFS() throws Exception {
116 FileSystem fs = cluster.getFileSystem();
117
118 File jarFile1 = buildCoprocessorJar(cpName1);
119 File jarFile2 = buildCoprocessorJar(cpName2);
120
121
122 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
123 new Path(fs.getUri().toString() + Path.SEPARATOR));
124 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
125 jarFile1.getName();
126 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
127 assertTrue("Copy jar file to HDFS failed.",
128 fs.exists(pathOnHDFS1));
129 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
130
131 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
132 new Path(fs.getUri().toString() + Path.SEPARATOR));
133 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
134 jarFile2.getName();
135 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
136 assertTrue("Copy jar file to HDFS failed.",
137 fs.exists(pathOnHDFS2));
138 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
139
140
141 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
142 htd.addFamily(new HColumnDescriptor("test"));
143
144 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
145 "|" + Coprocessor.PRIORITY_USER);
146
147 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
148 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
149 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
150 if (admin.tableExists(tableName)) {
151 if (admin.isTableEnabled(tableName)) {
152 admin.disableTable(tableName);
153 }
154 admin.deleteTable(tableName);
155 }
156 CoprocessorClassLoader.clearCache();
157 byte[] startKey = {10, 63};
158 byte[] endKey = {12, 43};
159 admin.createTable(htd, startKey, endKey, 4);
160 waitForTable(htd.getTableName());
161
162
163 boolean foundTableRegion=false;
164 boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
165 Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
166 new HashMap<HRegion, Set<ClassLoader>>();
167 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
168 for (HRegion region:
169 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
170 if (region.getRegionNameAsString().startsWith(tableName)) {
171 foundTableRegion = true;
172 CoprocessorEnvironment env;
173 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
174 found1 = found1 && (env != null);
175 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
176 found2 = found2 && (env != null);
177 if (env != null) {
178 Configuration conf = env.getConfiguration();
179 found2_k1 = found2_k1 && (conf.get("k1") != null);
180 found2_k2 = found2_k2 && (conf.get("k2") != null);
181 found2_k3 = found2_k3 && (conf.get("k3") != null);
182 } else {
183 found2_k1 = found2_k2 = found2_k3 = false;
184 }
185 regionsActiveClassLoaders
186 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
187 }
188 }
189
190 assertTrue("No region was found for table " + tableName, foundTableRegion);
191 assertTrue("Class " + cpName1 + " was missing on a region", found1);
192 assertTrue("Class " + cpName2 + " was missing on a region", found2);
193 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
194 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
195 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
196
197 assertNotNull(jarFileOnHDFS1 + " was not cached",
198 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
199 assertNotNull(jarFileOnHDFS2 + " was not cached",
200 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
201
202 assertEquals("The number of cached classloaders should be equal to the number" +
203 " of external jar files",
204 2, CoprocessorClassLoader.getAllCached().size());
205
206 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
207 CoprocessorClassLoader.getAllCached());
208 for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
209 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
210 + " ClassLoader Cache:" + externalClassLoaders
211 + " Region ClassLoaders:" + regionCP.getValue(),
212 externalClassLoaders.containsAll(regionCP.getValue()));
213 }
214 }
215
216 private String getLocalPath(File file) {
217 return new Path(file.toURI()).toString();
218 }
219
220 @Test
221
222 public void testClassLoadingFromLocalFS() throws Exception {
223 File jarFile = buildCoprocessorJar(cpName3);
224
225
226 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
227 htd.addFamily(new HColumnDescriptor("test"));
228 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
229 Coprocessor.PRIORITY_USER);
230 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
231 admin.createTable(htd);
232 waitForTable(htd.getTableName());
233
234
235 boolean found = false;
236 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
237 for (HRegion region:
238 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
239 if (region.getRegionNameAsString().startsWith(cpName3)) {
240 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
241 }
242 }
243 assertTrue("Class " + cpName3 + " was missing on a region", found);
244 }
245
246 @Test
247
248 public void testPrivateClassLoader() throws Exception {
249 File jarFile = buildCoprocessorJar(cpName4);
250
251
252 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
253 htd.addFamily(new HColumnDescriptor("test"));
254 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
255 Coprocessor.PRIORITY_USER);
256 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
257 admin.createTable(htd);
258 waitForTable(htd.getTableName());
259
260
261 boolean found = false;
262 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
263 for (HRegion region:
264 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
265 if (region.getRegionNameAsString().startsWith(cpName4)) {
266 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
267 if (cp != null) {
268 found = true;
269 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
270 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
271 }
272 }
273 }
274 assertTrue("Class " + cpName4 + " was missing on a region", found);
275 }
276
277 @Test
278
279
280 public void testHBase3810() throws Exception {
281
282
283 File jarFile1 = buildCoprocessorJar(cpName1);
284 File jarFile2 = buildCoprocessorJar(cpName2);
285 File jarFile5 = buildCoprocessorJar(cpName5);
286 File jarFile6 = buildCoprocessorJar(cpName6);
287
288 String cpKey1 = "COPROCESSOR$1";
289 String cpKey2 = " Coprocessor$2 ";
290 String cpKey3 = " coprocessor$03 ";
291
292 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
293 Coprocessor.PRIORITY_USER;
294 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
295
296 String cpValue3 =
297 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
298
299
300 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
301 htd.addFamily(new HColumnDescriptor("test"));
302
303
304 htd.setValue(cpKey1, cpValue1);
305 htd.setValue(cpKey2, cpValue2);
306 htd.setValue(cpKey3, cpValue3);
307
308
309 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
310 Coprocessor.PRIORITY_USER, null);
311 Map<String, String> kvs = new HashMap<String, String>();
312 kvs.put("k1", "v1");
313 kvs.put("k2", "v2");
314 kvs.put("k3", "v3");
315 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
316 Coprocessor.PRIORITY_USER, kvs);
317
318 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
319 if (admin.tableExists(tableName)) {
320 if (admin.isTableEnabled(tableName)) {
321 admin.disableTable(tableName);
322 }
323 admin.deleteTable(tableName);
324 }
325 admin.createTable(htd);
326 waitForTable(htd.getTableName());
327
328
329 boolean found_2 = false, found_1 = false, found_3 = false,
330 found_5 = false, found_6 = false;
331 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
332 found6_k4 = false;
333
334 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
335 for (HRegion region:
336 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
337 if (region.getRegionNameAsString().startsWith(tableName)) {
338 found_1 = found_1 ||
339 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
340 found_2 = found_2 ||
341 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
342 found_3 = found_3 ||
343 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
344 != null);
345 found_5 = found_5 ||
346 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
347
348 CoprocessorEnvironment env =
349 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
350 if (env != null) {
351 found_6 = true;
352 Configuration conf = env.getConfiguration();
353 found6_k1 = conf.get("k1") != null;
354 found6_k2 = conf.get("k2") != null;
355 found6_k3 = conf.get("k3") != null;
356 }
357 }
358 }
359
360 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
361 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
362 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
363 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
364 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
365
366 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
367 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
368 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
369 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
370 }
371
372 @Test
373 public void testClassLoadingFromLibDirInJar() throws Exception {
374 loadingClassFromLibDirInJar("/lib/");
375 }
376
377 @Test
378 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
379 loadingClassFromLibDirInJar("lib/");
380 }
381
382 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
383 FileSystem fs = cluster.getFileSystem();
384
385 File innerJarFile1 = buildCoprocessorJar(cpName1);
386 File innerJarFile2 = buildCoprocessorJar(cpName2);
387 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
388
389 ClassLoaderTestHelper.addJarFilesToJar(
390 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
391
392
393 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
394 new Path(fs.getUri().toString() + Path.SEPARATOR));
395 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
396 outerJarFile.getName();
397 assertTrue("Copy jar file to HDFS failed.",
398 fs.exists(new Path(jarFileOnHDFS)));
399 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
400
401
402 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
403 htd.addFamily(new HColumnDescriptor("test"));
404
405 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
406 "|" + Coprocessor.PRIORITY_USER);
407
408 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
409 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
410 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
411 if (admin.tableExists(tableName)) {
412 if (admin.isTableEnabled(tableName)) {
413 admin.disableTable(tableName);
414 }
415 admin.deleteTable(tableName);
416 }
417 admin.createTable(htd);
418 waitForTable(htd.getTableName());
419
420
421 boolean found1 = false, found2 = false, found2_k1 = false,
422 found2_k2 = false, found2_k3 = false;
423 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
424 for (HRegion region:
425 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
426 if (region.getRegionNameAsString().startsWith(tableName)) {
427 CoprocessorEnvironment env;
428 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
429 if (env != null) {
430 found1 = true;
431 }
432 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
433 if (env != null) {
434 found2 = true;
435 Configuration conf = env.getConfiguration();
436 found2_k1 = conf.get("k1") != null;
437 found2_k2 = conf.get("k2") != null;
438 found2_k3 = conf.get("k3") != null;
439 }
440 }
441 }
442 assertTrue("Class " + cpName1 + " was missing on a region", found1);
443 assertTrue("Class " + cpName2 + " was missing on a region", found2);
444 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
445 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
446 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
447 }
448
449 @Test
450 public void testRegionServerCoprocessorsReported() throws Exception {
451
452
453
454 assertAllRegionServers(regionServerSystemCoprocessors,null);
455 }
456
457
458
459
460
461
462
463
464
465
466 Map<ServerName, ServerLoad> serversForTable(String tableName) {
467 Map<ServerName, ServerLoad> serverLoadHashMap =
468 new HashMap<ServerName, ServerLoad>();
469 for(Map.Entry<ServerName,ServerLoad> server:
470 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
471 getOnlineServers().entrySet()) {
472 for( Map.Entry<byte[], RegionLoad> region:
473 server.getValue().getRegionsLoad().entrySet()) {
474 if (region.getValue().getNameAsString().equals(tableName)) {
475
476 serverLoadHashMap.put(server.getKey(),server.getValue());
477
478 break;
479 }
480 }
481 }
482 return serverLoadHashMap;
483 }
484
485 void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
486 throws InterruptedException {
487 Map<ServerName, ServerLoad> servers;
488 String[] actualCoprocessors = null;
489 boolean success = false;
490 for(int i = 0; i < 5; i++) {
491 if (tableName == null) {
492
493 servers =
494 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
495 getOnlineServers();
496 } else {
497 servers = serversForTable(tableName);
498 }
499 boolean any_failed = false;
500 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
501 actualCoprocessors = server.getValue().getRsCoprocessors();
502 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
503 LOG.debug("failed comparison: actual: " +
504 Arrays.toString(actualCoprocessors) +
505 " ; expected: " + Arrays.toString(expectedCoprocessors));
506 any_failed = true;
507 break;
508 }
509 }
510 if (any_failed == false) {
511 success = true;
512 break;
513 }
514 LOG.debug("retrying after failed comparison: " + i);
515 Thread.sleep(1000);
516 }
517 assertTrue(success);
518 }
519
520 @Test
521 public void testMasterCoprocessorsReported() {
522
523
524
525 final String loadedMasterCoprocessorsVerify =
526 "[" + masterCoprocessor.getSimpleName() + "]";
527 String loadedMasterCoprocessors =
528 java.util.Arrays.toString(
529 TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
530 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
531 }
532
533 @Test
534 public void testFindCoprocessors() {
535
536 CoprocessorHost masterCpHost = TEST_UTIL.getHBaseCluster().getMaster().getCoprocessorHost();
537
538 List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
539
540 assertTrue(masterObservers != null && masterObservers.size() > 0);
541 assertEquals(masterCoprocessor.getSimpleName(),
542 masterObservers.get(0).getClass().getSimpleName());
543 }
544
545 private void waitForTable(TableName name) throws InterruptedException, IOException {
546
547 TEST_UTIL.waitTableEnabled(name.getName());
548
549 Thread.sleep(1000);
550 }
551
552 }
553