1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import java.util.Set;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.HBaseConfiguration;
28 import org.apache.hadoop.hbase.IntegrationTestBase;
29 import org.apache.hadoop.hbase.IntegrationTestingUtility;
30 import org.apache.hadoop.hbase.testclassification.IntegrationTests;
31 import org.apache.hadoop.hbase.TableName;
32 import org.apache.hadoop.hbase.util.Bytes;
33 import org.apache.hadoop.util.ToolRunner;
34 import org.junit.After;
35 import org.junit.Before;
36 import org.junit.experimental.categories.Category;
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68 @Category(IntegrationTests.class)
69
70 public class IntegrationTestTableSnapshotInputFormat extends IntegrationTestBase {
71
72 private static final Log LOG = LogFactory.getLog(IntegrationTestTableSnapshotInputFormat.class);
73
74 private static final String TABLE_NAME_KEY = "IntegrationTestTableSnapshotInputFormat.table";
75 private static final String DEFAULT_TABLE_NAME = "IntegrationTestTableSnapshotInputFormat";
76
77 private static final String SNAPSHOT_NAME_KEY = "IntegrationTestTableSnapshotInputFormat.snapshot";
78
79 private static final String MR_IMPLEMENTATION_KEY =
80 "IntegrationTestTableSnapshotInputFormat.API";
81 private static final String MAPRED_IMPLEMENTATION = "mapred";
82 private static final String MAPREDUCE_IMPLEMENTATION = "mapreduce";
83
84 private static final String NUM_REGIONS_KEY =
85 "IntegrationTestTableSnapshotInputFormat.numRegions";
86 private static final int DEFAULT_NUM_REGIONS = 32;
87 private static final String TABLE_DIR_KEY = "IntegrationTestTableSnapshotInputFormat.tableDir";
88
89 private static final byte[] START_ROW = Bytes.toBytes("bbb");
90 private static final byte[] END_ROW = Bytes.toBytes("yyy");
91
92
93
94 private static final byte[] MAPRED_START_ROW = Bytes.toBytes("aaa");
95 private static final byte[] MAPRED_END_ROW = Bytes.toBytes("zz{");
96
97 private IntegrationTestingUtility util;
98
99 @Override
100 public void setConf(Configuration conf) {
101 super.setConf(conf);
102 util = getTestingUtil(conf);
103 }
104
105 @Override
106 @Before
107 public void setUp() throws Exception {
108 super.setUp();
109 util = getTestingUtil(getConf());
110 util.initializeCluster(1);
111 this.setConf(util.getConfiguration());
112 }
113
114 @Override
115 @After
116 public void cleanUp() throws Exception {
117 util.restoreCluster();
118 }
119
120 @Override
121 public void setUpCluster() throws Exception {
122 }
123
124 @Override
125 public int runTestFromCommandLine() throws Exception {
126 Configuration conf = getConf();
127 TableName tableName = TableName.valueOf(conf.get(TABLE_NAME_KEY, DEFAULT_TABLE_NAME));
128 String snapshotName = conf.get(SNAPSHOT_NAME_KEY, tableName.getQualifierAsString()
129 + "_snapshot_" + System.currentTimeMillis());
130 int numRegions = conf.getInt(NUM_REGIONS_KEY, DEFAULT_NUM_REGIONS);
131 String tableDirStr = conf.get(TABLE_DIR_KEY);
132 Path tableDir;
133 if (tableDirStr == null) {
134 tableDir = util.getDataTestDirOnTestFS(tableName.getQualifierAsString());
135 } else {
136 tableDir = new Path(tableDirStr);
137 }
138
139 final String mr = conf.get(MR_IMPLEMENTATION_KEY, MAPREDUCE_IMPLEMENTATION);
140 if (mr.equalsIgnoreCase(MAPREDUCE_IMPLEMENTATION)) {
141
142
143
144
145
146
147
148
149 LOG.debug("Running job with mapreduce API.");
150 int expectedNumSplits = numRegions > 2 ? numRegions - 2 : numRegions;
151
152 org.apache.hadoop.hbase.mapreduce.TestTableSnapshotInputFormat.doTestWithMapReduce(util,
153 tableName, snapshotName, START_ROW, END_ROW, tableDir, numRegions,
154 expectedNumSplits, false);
155 } else if (mr.equalsIgnoreCase(MAPRED_IMPLEMENTATION)) {
156
157
158
159
160
161
162
163 LOG.debug("Running job with mapred API.");
164 int expectedNumSplits = numRegions;
165
166 org.apache.hadoop.hbase.mapred.TestTableSnapshotInputFormat.doTestWithMapReduce(util,
167 tableName, snapshotName, MAPRED_START_ROW, MAPRED_END_ROW, tableDir, numRegions,
168 expectedNumSplits, false);
169 } else {
170 throw new IllegalArgumentException("Unrecognized mapreduce implementation: " + mr +".");
171 }
172
173 return 0;
174 }
175
176 @Override
177 public String getTablename() {
178 return null;
179 }
180
181 @Override
182 protected Set<String> getColumnFamilies() {
183 return null;
184 }
185
186 public static void main(String[] args) throws Exception {
187 Configuration conf = HBaseConfiguration.create();
188 IntegrationTestingUtility.setUseDistributedCluster(conf);
189 int ret = ToolRunner.run(conf, new IntegrationTestTableSnapshotInputFormat(), args);
190 System.exit(ret);
191 }
192
193 }