1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Random;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.TableName;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.testclassification.LargeTests;
34 import org.apache.hadoop.hbase.client.HBaseAdmin;
35 import org.apache.hadoop.hbase.client.HTable;
36 import org.apache.hadoop.hbase.client.Put;
37 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.AfterClass;
40 import org.junit.BeforeClass;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43
44
45 @Category(LargeTests.class)
46 public class TestCompactionState {
47 final static Log LOG = LogFactory.getLog(TestCompactionState.class);
48 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49 private final static Random random = new Random();
50
51 @BeforeClass
52 public static void setUpBeforeClass() throws Exception {
53 TEST_UTIL.startMiniCluster();
54 }
55
56 @AfterClass
57 public static void tearDownAfterClass() throws Exception {
58 TEST_UTIL.shutdownMiniCluster();
59 }
60
61 @Test(timeout=600000)
62 public void testMajorCompaction() throws IOException, InterruptedException {
63 compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
64 }
65
66 @Test(timeout=600000)
67 public void testMinorCompaction() throws IOException, InterruptedException {
68 compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
69 }
70
71 @Test(timeout=600000)
72 public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
73 compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
74 }
75
76 @Test(timeout=600000)
77 public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
78 compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
79 }
80
81 @Test
82 public void testInvalidColumnFamily() throws IOException, InterruptedException {
83 byte [] table = Bytes.toBytes("testInvalidColumnFamily");
84 byte [] family = Bytes.toBytes("family");
85 byte [] fakecf = Bytes.toBytes("fakecf");
86 boolean caughtMinorCompact = false;
87 boolean caughtMajorCompact = false;
88 HTable ht = null;
89 try {
90 ht = TEST_UTIL.createTable(table, family);
91 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
92 try {
93 admin.compact(table, fakecf);
94 } catch (IOException ioe) {
95 caughtMinorCompact = true;
96 }
97 try {
98 admin.majorCompact(table, fakecf);
99 } catch (IOException ioe) {
100 caughtMajorCompact = true;
101 }
102 } finally {
103 if (ht != null) {
104 TEST_UTIL.deleteTable(table);
105 }
106 assertTrue(caughtMinorCompact);
107 assertTrue(caughtMajorCompact);
108 }
109 }
110
111
112
113
114
115
116
117
118
119
120
121
122 private void compaction(final String tableName, final int flushes,
123 final CompactionState expectedState, boolean singleFamily)
124 throws IOException, InterruptedException {
125
126 TableName table =
127 TableName.valueOf(tableName);
128 byte [] family = Bytes.toBytes("family");
129 byte [][] families =
130 {family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
131 HTable ht = null;
132 try {
133 ht = TEST_UTIL.createTable(table, families);
134 loadData(ht, families, 3000, flushes);
135 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
136 List<HRegion> regions = rs.getOnlineRegions(table);
137 int countBefore = countStoreFilesInFamilies(regions, families);
138 int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
139 assertTrue(countBefore > 0);
140 HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
141 if (expectedState == CompactionState.MINOR) {
142 if (singleFamily) {
143 admin.compact(table.getName(), family);
144 } else {
145 admin.compact(table.getName());
146 }
147 } else {
148 if (singleFamily) {
149 admin.majorCompact(table.getName(), family);
150 } else {
151 admin.majorCompact(table.getName());
152 }
153 }
154 long curt = System.currentTimeMillis();
155 long waitTime = 5000;
156 long endt = curt + waitTime;
157 CompactionState state = admin.getCompactionState(table.getName());
158 while (state == CompactionState.NONE && curt < endt) {
159 Thread.sleep(10);
160 state = admin.getCompactionState(table.getName());
161 curt = System.currentTimeMillis();
162 }
163
164
165 if (expectedState != state) {
166 for (HRegion region: regions) {
167 state = region.getCompactionState();
168 assertEquals(CompactionState.NONE, state);
169 }
170 } else {
171
172 state = admin.getCompactionState(table.getName());
173 while (state != CompactionState.NONE && curt < endt) {
174 Thread.sleep(10);
175 state = admin.getCompactionState(table.getName());
176 }
177
178 assertEquals(CompactionState.NONE, state);
179 }
180 int countAfter = countStoreFilesInFamilies(regions, families);
181 int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
182 assertTrue(countAfter < countBefore);
183 if (!singleFamily) {
184 if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
185 else assertTrue(families.length < countAfter);
186 } else {
187 int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
188
189 assertTrue(singleFamDiff == (countBefore - countAfter));
190 if (expectedState == CompactionState.MAJOR) {
191 assertTrue(1 == countAfterSingleFamily);
192 } else {
193 assertTrue(1 < countAfterSingleFamily);
194 }
195 }
196 } finally {
197 if (ht != null) {
198 TEST_UTIL.deleteTable(table);
199 }
200 }
201 }
202
203 private static int countStoreFilesInFamily(
204 List<HRegion> regions, final byte[] family) {
205 return countStoreFilesInFamilies(regions, new byte[][]{family});
206 }
207
208 private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
209 int count = 0;
210 for (HRegion region: regions) {
211 count += region.getStoreFileList(families).size();
212 }
213 return count;
214 }
215
216 private static void loadData(final HTable ht, final byte[][] families,
217 final int rows, final int flushes) throws IOException {
218 List<Put> puts = new ArrayList<Put>(rows);
219 byte[] qualifier = Bytes.toBytes("val");
220 for (int i = 0; i < flushes; i++) {
221 for (int k = 0; k < rows; k++) {
222 byte[] row = Bytes.toBytes(random.nextLong());
223 Put p = new Put(row);
224 for (int j = 0; j < families.length; ++j) {
225 p.add(families[ j ], qualifier, row);
226 }
227 puts.add(p);
228 }
229 ht.put(puts);
230 ht.flushCommits();
231 TEST_UTIL.flush();
232 puts.clear();
233 }
234 }
235 }