1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.util;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.Cell;
30 import org.apache.hadoop.hbase.CellUtil;
31 import org.apache.hadoop.hbase.HBaseTestCase;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.HTableDescriptor;
37 import org.apache.hadoop.hbase.testclassification.LargeTests;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.regionserver.HRegion;
43 import org.apache.hadoop.hbase.regionserver.InternalScanner;
44 import org.apache.hadoop.hbase.regionserver.wal.HLog;
45 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
46 import org.apache.hadoop.hdfs.MiniDFSCluster;
47 import org.apache.hadoop.util.ToolRunner;
48 import org.junit.experimental.categories.Category;
49
50
51 @Category(LargeTests.class)
52 public class TestMergeTool extends HBaseTestCase {
53 static final Log LOG = LogFactory.getLog(TestMergeTool.class);
54 HBaseTestingUtility TEST_UTIL;
55
56 static final byte [] FAMILY = Bytes.toBytes("contents");
57 static final byte [] QUALIFIER = Bytes.toBytes("dc");
58
59 private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
60 private final HRegion[] regions = new HRegion[5];
61 private HTableDescriptor desc;
62 private byte [][][] rows;
63 private MiniDFSCluster dfsCluster = null;
64
65 @Override
66 public void setUp() throws Exception {
67
68 this.conf.setLong("hbase.zookeeper.recoverable.waittime", 10);
69
70
71
72
73 this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 10001);
74
75 this.conf.set("hbase.hstore.compactionThreshold", "2");
76
77
78 this.desc = new HTableDescriptor(org.apache.hadoop.hbase.TableName.valueOf("TestMergeTool"));
79 this.desc.addFamily(new HColumnDescriptor(FAMILY));
80
81
82
83
84
85 sourceRegions[0] = new HRegionInfo(this.desc.getTableName(),
86 Bytes.toBytes("row_0200"),
87 Bytes.toBytes("row_0300"));
88
89
90
91 sourceRegions[1] =
92 new HRegionInfo(this.desc.getTableName(),
93 Bytes.toBytes("row_0250"),
94 Bytes.toBytes("row_0400"));
95
96
97
98 sourceRegions[2] =
99 new HRegionInfo(this.desc.getTableName(),
100 Bytes.toBytes("row_0100"),
101 Bytes.toBytes("row_0200"));
102
103
104
105
106 sourceRegions[3] =
107 new HRegionInfo(this.desc.getTableName(),
108 Bytes.toBytes("row_0500"),
109 Bytes.toBytes("row_0600"));
110
111
112 sourceRegions[4] =
113 new HRegionInfo(this.desc.getTableName(),
114 HConstants.EMPTY_BYTE_ARRAY,
115 HConstants.EMPTY_BYTE_ARRAY);
116
117
118
119
120 this.rows = new byte [5][][];
121 this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
122 this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
123 "row_035" });
124 this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
125 "row_0175", "row_0175"});
126 this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
127 "row_0560", "row_0560", "row_0560"});
128 this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
129 "row_1000", "row_1000", "row_1000", "row_1000" });
130
131
132 TEST_UTIL = new HBaseTestingUtility(conf);
133 this.dfsCluster = TEST_UTIL.startMiniDFSCluster(2);
134 this.fs = this.dfsCluster.getFileSystem();
135 System.out.println("fs=" + this.fs);
136 FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
137 Path parentdir = fs.getHomeDirectory();
138 FSUtils.setRootDir(conf, parentdir);
139 fs.mkdirs(parentdir);
140 FSUtils.setVersion(fs, parentdir);
141
142
143
144
145 super.setUp();
146 try {
147
148 createMetaRegion();
149 new FSTableDescriptors(conf, this.fs, this.testDir).createTableDescriptor(this.desc);
150
151
152
153 for (int i = 0; i < sourceRegions.length; i++) {
154 regions[i] =
155 HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf,
156 this.desc);
157
158
159
160 for (int j = 0; j < rows[i].length; j++) {
161 byte [] row = rows[i][j];
162 Put put = new Put(row);
163 put.add(FAMILY, QUALIFIER, row);
164 regions[i].put(put);
165 }
166 HRegion.addRegionToMETA(meta, regions[i]);
167 }
168
169 closeRootAndMeta();
170
171 } catch (Exception e) {
172 TEST_UTIL.shutdownMiniCluster();
173 throw e;
174 }
175 }
176
177 @Override
178 public void tearDown() throws Exception {
179 super.tearDown();
180 for (int i = 0; i < sourceRegions.length; i++) {
181 HRegion r = regions[i];
182 if (r != null) {
183 HRegion.closeHRegion(r);
184 }
185 }
186 TEST_UTIL.shutdownMiniCluster();
187 }
188
189
190
191
192
193
194
195
196
197
198 private HRegion mergeAndVerify(final String msg, final String regionName1,
199 final String regionName2, final HLog log, final int upperbound)
200 throws Exception {
201 Merge merger = new Merge(this.conf);
202 LOG.info(msg);
203 LOG.info("fs2=" + this.conf.get("fs.defaultFS"));
204 int errCode = ToolRunner.run(this.conf, merger,
205 new String[] {this.desc.getTableName().getNameAsString(), regionName1, regionName2}
206 );
207 assertTrue("'" + msg + "' failed with errCode " + errCode, errCode == 0);
208 HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
209
210
211
212 HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf);
213 verifyMerge(merged, upperbound);
214 merged.close();
215 LOG.info("Verified " + msg);
216 return merged;
217 }
218
219 private void verifyMerge(final HRegion merged, final int upperbound)
220 throws IOException {
221
222 Scan scan = new Scan();
223 scan.addFamily(FAMILY);
224 InternalScanner scanner = merged.getScanner(scan);
225 try {
226 List<Cell> testRes = null;
227 while (true) {
228 testRes = new ArrayList<Cell>();
229 boolean hasNext = scanner.next(testRes);
230 if (!hasNext) {
231 break;
232 }
233 }
234 } finally {
235 scanner.close();
236 }
237
238
239
240 for (int i = 0; i < upperbound; i++) {
241 for (int j = 0; j < rows[i].length; j++) {
242 Get get = new Get(rows[i][j]);
243 get.addFamily(FAMILY);
244 Result result = merged.get(get);
245 assertEquals(1, result.size());
246 byte [] bytes = CellUtil.cloneValue(result.rawCells()[0]);
247 assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes);
248 assertTrue(Bytes.equals(bytes, rows[i][j]));
249 }
250 }
251 }
252
253
254
255
256
257 public void testMergeTool() throws Exception {
258
259
260 for (int i = 0; i < regions.length; i++) {
261 for (int j = 0; j < rows[i].length; j++) {
262 Get get = new Get(rows[i][j]);
263 get.addFamily(FAMILY);
264 Result result = regions[i].get(get);
265 byte [] bytes = CellUtil.cloneValue(result.rawCells()[0]);
266 assertNotNull(bytes);
267 assertTrue(Bytes.equals(bytes, rows[i][j]));
268 }
269
270 HRegion.closeHRegion(regions[i]);
271 }
272
273
274 Path logPath = new Path("/tmp");
275 String logName = HConstants.HREGION_LOGDIR_NAME + "_"
276 + System.currentTimeMillis();
277 LOG.info("Creating log " + logPath.toString() + "/" + logName);
278
279 HLog log = HLogFactory.createHLog(this.fs, logPath,
280 logName, this.conf);
281
282 try {
283
284 HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
285 this.sourceRegions[0].getRegionNameAsString(),
286 this.sourceRegions[1].getRegionNameAsString(), log, 2);
287
288
289 merged = mergeAndVerify("merging regions 0+1 and 2",
290 merged.getRegionInfo().getRegionNameAsString(),
291 this.sourceRegions[2].getRegionNameAsString(), log, 3);
292
293
294 merged = mergeAndVerify("merging regions 0+1+2 and 3",
295 merged.getRegionInfo().getRegionNameAsString(),
296 this.sourceRegions[3].getRegionNameAsString(), log, 4);
297
298
299 merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
300 merged.getRegionInfo().getRegionNameAsString(),
301 this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
302 } finally {
303 log.closeAndDelete();
304 }
305 }
306
307 }
308