1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.hbase.*;
24 import org.apache.hadoop.hbase.client.*;
25 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26 import org.apache.hadoop.hbase.testclassification.MediumTests;
27 import org.apache.hadoop.hbase.util.Bytes;
28 import org.apache.hadoop.hbase.util.Threads;
29 import org.apache.log4j.Level;
30 import org.apache.log4j.Logger;
31 import org.junit.AfterClass;
32 import org.junit.BeforeClass;
33 import org.junit.Test;
34 import org.junit.experimental.categories.Category;
35 import static org.junit.Assert.*;
36
37 import java.io.IOException;
38 import java.util.ArrayList;
39 import java.util.List;
40
41
42 @Category(MediumTests.class)
43 public class TestRegionServerMetrics {
44 private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
45 private static MetricsAssertHelper metricsHelper;
46
47 static {
48 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
49 }
50
51 private static MiniHBaseCluster cluster;
52 private static HRegionServer rs;
53 private static Configuration conf;
54 private static HBaseTestingUtility TEST_UTIL;
55 private static MetricsRegionServer metricsRegionServer;
56 private static MetricsRegionServerSource serverSource;
57
58 @BeforeClass
59 public static void startCluster() throws Exception {
60 metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
61 TEST_UTIL = new HBaseTestingUtility();
62 conf = TEST_UTIL.getConfiguration();
63 conf.getLong("hbase.splitlog.max.resubmit", 0);
64
65 conf.setInt("zookeeper.recovery.retry", 0);
66 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
67
68 TEST_UTIL.startMiniCluster(1, 1);
69 cluster = TEST_UTIL.getHBaseCluster();
70
71 cluster.waitForActiveAndReadyMaster();
72
73 while (cluster.getLiveRegionServerThreads().size() < 1) {
74 Threads.sleep(100);
75 }
76
77 rs = cluster.getRegionServer(0);
78 metricsRegionServer = rs.getMetrics();
79 serverSource = metricsRegionServer.getMetricsSource();
80 }
81
82 @AfterClass
83 public static void after() throws Exception {
84 if (TEST_UTIL != null) {
85 TEST_UTIL.shutdownMiniCluster();
86 }
87 }
88
89 @Test(timeout = 300000)
90 public void testRegionCount() throws Exception {
91 String regionMetricsKey = "regionCount";
92 long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
93
94 TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
95 metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
96 }
97
98 @Test
99 public void testLocalFiles() throws Exception {
100 metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
101 }
102
103 @Test
104 public void testRequestCount() throws Exception {
105 String tableNameString = "testRequestCount";
106 byte[] tName = Bytes.toBytes(tableNameString);
107 byte[] cfName = Bytes.toBytes("d");
108 byte[] row = Bytes.toBytes("rk");
109 byte[] qualifier = Bytes.toBytes("qual");
110 byte[] initValue = Bytes.toBytes("Value");
111 byte[] nextValue = Bytes.toBytes("NEXT VAL");
112
113
114 TEST_UTIL.createTable(tName, cfName);
115
116 new HTable(conf, tName).close();
117
118
119 HTable table = new HTable(conf, tName);
120 Put p = new Put(row);
121 p.add(cfName, qualifier, initValue);
122 table.put(p);
123
124 metricsRegionServer.getRegionServerWrapper().forceRecompute();
125 long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
126 long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
127 long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
128
129 for (int i=0; i< 30; i++) {
130 table.put(p);
131 }
132
133 metricsRegionServer.getRegionServerWrapper().forceRecompute();
134 metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
135 metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
136 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
137
138 Get g = new Get(row);
139 for (int i=0; i< 10; i++) {
140 table.get(g);
141 }
142
143 metricsRegionServer.getRegionServerWrapper().forceRecompute();
144 metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
145 metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
146 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
147
148 for ( HRegionInfo i:table.getRegionLocations().keySet()) {
149 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
150 .getMetrics()
151 .getSource()
152 .getAggregateSource();
153 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
154 "_table_"+tableNameString +
155 "_region_" + i.getEncodedName()+
156 "_metric";
157 metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
158 metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
159 }
160
161
162 metricsRegionServer.getRegionServerWrapper().forceRecompute();
163 metricsHelper.assertCounter("totalRequestCount", requests + 40 + 3, serverSource);
164 metricsHelper.assertCounter("readRequestCount", readRequests + 10 + 1, serverSource);
165
166
167 List<Get> gets = new ArrayList<Get>();
168 for (int i=0; i< 10; i++) {
169 gets.add(new Get(row));
170 }
171 table.get(gets);
172
173 metricsRegionServer.getRegionServerWrapper().forceRecompute();
174 metricsHelper.assertCounter("totalRequestCount", requests + 50 + 3, serverSource);
175 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
176 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
177
178 table.setAutoFlushTo(false);
179 for (int i=0; i< 30; i++) {
180 table.put(p);
181 }
182 table.flushCommits();
183
184 metricsRegionServer.getRegionServerWrapper().forceRecompute();
185 metricsHelper.assertCounter("totalRequestCount", requests + 80 + 3, serverSource);
186 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
187 metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
188
189 table.close();
190 }
191
192 @Test
193 public void testMutationsWithoutWal() throws Exception {
194 byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
195 byte[] cf = Bytes.toBytes("d");
196 byte[] row = Bytes.toBytes("rk");
197 byte[] qualifier = Bytes.toBytes("qual");
198 byte[] val = Bytes.toBytes("Value");
199
200 metricsRegionServer.getRegionServerWrapper().forceRecompute();
201
202 TEST_UTIL.createTable(tableName, cf);
203
204 HTable t = new HTable(conf, tableName);
205
206 Put p = new Put(row);
207 p.add(cf, qualifier, val);
208 p.setDurability(Durability.SKIP_WAL);
209
210 t.put(p);
211 t.flushCommits();
212
213 metricsRegionServer.getRegionServerWrapper().forceRecompute();
214 metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
215 long minLength = row.length + cf.length + qualifier.length + val.length;
216 metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
217
218 t.close();
219 }
220
221 @Test
222 public void testStoreCount() throws Exception {
223 byte[] tableName = Bytes.toBytes("testStoreCount");
224 byte[] cf = Bytes.toBytes("d");
225 byte[] row = Bytes.toBytes("rk");
226 byte[] qualifier = Bytes.toBytes("qual");
227 byte[] val = Bytes.toBytes("Value");
228
229 metricsRegionServer.getRegionServerWrapper().forceRecompute();
230 long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
231 long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
232
233 TEST_UTIL.createTable(tableName, cf);
234
235
236 HTable t = new HTable(conf, tableName);
237 Put p = new Put(row);
238 p.add(cf, qualifier, val);
239 t.put(p);
240 t.flushCommits();
241 TEST_UTIL.getHBaseAdmin().flush(tableName);
242
243 metricsRegionServer.getRegionServerWrapper().forceRecompute();
244 metricsHelper.assertGauge("storeCount", stores +1, serverSource);
245 metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
246
247 t.close();
248 }
249
250 @Test
251 public void testCheckAndPutCount() throws Exception {
252 String tableNameString = "testCheckAndPutCount";
253 byte[] tableName = Bytes.toBytes(tableNameString);
254 byte[] cf = Bytes.toBytes("d");
255 byte[] row = Bytes.toBytes("rk");
256 byte[] qualifier = Bytes.toBytes("qual");
257 byte[] valOne = Bytes.toBytes("Value");
258 byte[] valTwo = Bytes.toBytes("ValueTwo");
259 byte[] valThree = Bytes.toBytes("ValueThree");
260
261 TEST_UTIL.createTable(tableName, cf);
262 HTable t = new HTable(conf, tableName);
263 Put p = new Put(row);
264 p.add(cf, qualifier, valOne);
265 t.put(p);
266 t.flushCommits();
267
268 Put pTwo = new Put(row);
269 pTwo.add(cf, qualifier, valTwo);
270 t.checkAndPut(row, cf, qualifier, valOne, pTwo);
271 t.flushCommits();
272
273 Put pThree = new Put(row);
274 pThree.add(cf, qualifier, valThree);
275 t.checkAndPut(row, cf, qualifier, valOne, pThree);
276 t.flushCommits();
277
278
279 metricsRegionServer.getRegionServerWrapper().forceRecompute();
280 metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
281 metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
282
283 t.close();
284 }
285
286 @Test
287 public void testIncrement() throws Exception {
288 String tableNameString = "testIncrement";
289 byte[] tableName = Bytes.toBytes(tableNameString);
290 byte[] cf = Bytes.toBytes("d");
291 byte[] row = Bytes.toBytes("rk");
292 byte[] qualifier = Bytes.toBytes("qual");
293 byte[] val = Bytes.toBytes(0l);
294
295
296 TEST_UTIL.createTable(tableName, cf);
297 HTable t = new HTable(conf, tableName);
298
299 Put p = new Put(row);
300 p.add(cf, qualifier, val);
301 t.put(p);
302 t.flushCommits();
303
304 for(int count = 0; count< 13; count++) {
305 Increment inc = new Increment(row);
306 inc.addColumn(cf, qualifier, 100);
307 t.increment(inc);
308 }
309
310 t.flushCommits();
311
312 metricsRegionServer.getRegionServerWrapper().forceRecompute();
313 metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
314
315 t.close();
316 }
317
318 @Test
319 public void testAppend() throws Exception {
320 String tableNameString = "testAppend";
321 byte[] tableName = Bytes.toBytes(tableNameString);
322 byte[] cf = Bytes.toBytes("d");
323 byte[] row = Bytes.toBytes("rk");
324 byte[] qualifier = Bytes.toBytes("qual");
325 byte[] val = Bytes.toBytes("One");
326
327
328 TEST_UTIL.createTable(tableName, cf);
329 HTable t = new HTable(conf, tableName);
330
331 Put p = new Put(row);
332 p.add(cf, qualifier, val);
333 t.put(p);
334 t.flushCommits();
335
336 for(int count = 0; count< 73; count++) {
337 Append append = new Append(row);
338 append.add(cf, qualifier, Bytes.toBytes(",Test"));
339 t.append(append);
340 }
341
342 t.flushCommits();
343
344 metricsRegionServer.getRegionServerWrapper().forceRecompute();
345 metricsHelper.assertCounter("appendNumOps", 73, serverSource);
346
347 t.close();
348 }
349
350 @Test
351 public void testScanNext() throws IOException {
352 String tableNameString = "testScanNext";
353 byte[] tableName = Bytes.toBytes(tableNameString);
354 byte[] cf = Bytes.toBytes("d");
355 byte[] qualifier = Bytes.toBytes("qual");
356 byte[] val = Bytes.toBytes("One");
357
358
359 TEST_UTIL.createTable(tableName, cf);
360 HTable t = new HTable(conf, tableName);
361 t.setAutoFlush(false, true);
362 for (int insertCount =0; insertCount < 100; insertCount++) {
363 Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
364 p.add(cf, qualifier, val);
365 t.put(p);
366 }
367 t.flushCommits();
368
369 Scan s = new Scan();
370 s.setBatch(1);
371 s.setCaching(1);
372 ResultScanner resultScanners = t.getScanner(s);
373
374 for (int nextCount = 0; nextCount < 30; nextCount++) {
375 Result result = resultScanners.next();
376 assertNotNull(result);
377 assertEquals(1, result.size());
378 }
379 for ( HRegionInfo i:t.getRegionLocations().keySet()) {
380 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
381 .getMetrics()
382 .getSource()
383 .getAggregateSource();
384 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
385 "_table_"+tableNameString +
386 "_region_" + i.getEncodedName()+
387 "_metric";
388 metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
389 }
390 }
391 }