1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.FileNotFoundException;
22 import java.io.IOException;
23 import java.util.Comparator;
24 import java.util.HashSet;
25 import java.util.Map;
26 import java.util.TreeMap;
27 import java.util.concurrent.atomic.AtomicBoolean;
28 import java.util.concurrent.atomic.AtomicInteger;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.Chore;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.HColumnDescriptor;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.Server;
42 import org.apache.hadoop.hbase.backup.HFileArchiver;
43 import org.apache.hadoop.hbase.catalog.MetaEditor;
44 import org.apache.hadoop.hbase.catalog.MetaReader;
45 import org.apache.hadoop.hbase.client.MetaScanner;
46 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Pair;
52 import org.apache.hadoop.hbase.util.PairOfSameType;
53 import org.apache.hadoop.hbase.util.Triple;
54
55
56
57
58
59 @InterfaceAudience.Private
60 public class CatalogJanitor extends Chore {
61 private static final Log LOG = LogFactory.getLog(CatalogJanitor.class.getName());
62 private final Server server;
63 private final MasterServices services;
64 private AtomicBoolean enabled = new AtomicBoolean(true);
65 private AtomicBoolean alreadyRunning = new AtomicBoolean(false);
66
67 CatalogJanitor(final Server server, final MasterServices services) {
68 super("CatalogJanitor-" + server.getServerName().toShortString(),
69 server.getConfiguration().getInt("hbase.catalogjanitor.interval", 300000),
70 server);
71 this.server = server;
72 this.services = services;
73 }
74
75 @Override
76 protected boolean initialChore() {
77 try {
78 if (this.enabled.get()) scan();
79 } catch (IOException e) {
80 LOG.warn("Failed initial scan of catalog table", e);
81 return false;
82 }
83 return true;
84 }
85
86
87
88
89 public boolean setEnabled(final boolean enabled) {
90 return this.enabled.getAndSet(enabled);
91 }
92
93 boolean getEnabled() {
94 return this.enabled.get();
95 }
96
97 @Override
98 protected void chore() {
99 try {
100 if (this.enabled.get()) {
101 scan();
102 } else {
103 LOG.warn("CatalogJanitor disabled! Not running scan.");
104 }
105 } catch (IOException e) {
106 LOG.warn("Failed scan of catalog table", e);
107 }
108 }
109
110
111
112
113
114
115
116
117 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents()
118 throws IOException {
119 return getMergedRegionsAndSplitParents(null);
120 }
121
122
123
124
125
126
127
128
129
130
131
132 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> getMergedRegionsAndSplitParents(
133 final TableName tableName) throws IOException {
134 final boolean isTableSpecified = (tableName != null);
135
136 final AtomicInteger count = new AtomicInteger(0);
137
138
139 final Map<HRegionInfo, Result> splitParents =
140 new TreeMap<HRegionInfo, Result>(new SplitParentFirstComparator());
141 final Map<HRegionInfo, Result> mergedRegions = new TreeMap<HRegionInfo, Result>();
142
143
144 MetaScannerVisitor visitor = new MetaScanner.MetaScannerVisitorBase() {
145 @Override
146 public boolean processRow(Result r) throws IOException {
147 if (r == null || r.isEmpty()) return true;
148 count.incrementAndGet();
149 HRegionInfo info = HRegionInfo.getHRegionInfo(r);
150 if (info == null) return true;
151 if (isTableSpecified
152 && info.getTable().compareTo(tableName) > 0) {
153
154 return false;
155 }
156 if (info.isSplitParent()) splitParents.put(info, r);
157 if (r.getValue(HConstants.CATALOG_FAMILY, HConstants.MERGEA_QUALIFIER) != null) {
158 mergedRegions.put(info, r);
159 }
160
161 return true;
162 }
163 };
164
165
166
167 MetaScanner.metaScan(server.getConfiguration(), null, visitor, tableName);
168
169 return new Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>>(
170 count.get(), mergedRegions, splitParents);
171 }
172
173
174
175
176
177
178
179
180
181
182
183 boolean cleanMergeRegion(final HRegionInfo mergedRegion,
184 final HRegionInfo regionA, final HRegionInfo regionB) throws IOException {
185 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
186 Path rootdir = this.services.getMasterFileSystem().getRootDir();
187 Path tabledir = FSUtils.getTableDir(rootdir, mergedRegion.getTable());
188 HTableDescriptor htd = getTableDescriptor(mergedRegion.getTable());
189 HRegionFileSystem regionFs = null;
190 try {
191 regionFs = HRegionFileSystem.openRegionFromFileSystem(
192 this.services.getConfiguration(), fs, tabledir, mergedRegion, true);
193 } catch (IOException e) {
194 LOG.warn("Merged region does not exist: " + mergedRegion.getEncodedName());
195 }
196 if (regionFs == null || !regionFs.hasReferences(htd)) {
197 LOG.debug("Deleting region " + regionA.getRegionNameAsString() + " and "
198 + regionB.getRegionNameAsString()
199 + " from fs because merged region no longer holds references");
200 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionA);
201 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, regionB);
202 MetaEditor.deleteMergeQualifiers(server.getCatalogTracker(), mergedRegion);
203 return true;
204 }
205 return false;
206 }
207
208
209
210
211
212
213
214 int scan() throws IOException {
215 try {
216 if (!alreadyRunning.compareAndSet(false, true)) {
217 return 0;
218 }
219 Triple<Integer, Map<HRegionInfo, Result>, Map<HRegionInfo, Result>> scanTriple =
220 getMergedRegionsAndSplitParents();
221 int count = scanTriple.getFirst();
222
223
224
225 int mergeCleaned = 0;
226 Map<HRegionInfo, Result> mergedRegions = scanTriple.getSecond();
227 for (Map.Entry<HRegionInfo, Result> e : mergedRegions.entrySet()) {
228 HRegionInfo regionA = HRegionInfo.getHRegionInfo(e.getValue(),
229 HConstants.MERGEA_QUALIFIER);
230 HRegionInfo regionB = HRegionInfo.getHRegionInfo(e.getValue(),
231 HConstants.MERGEB_QUALIFIER);
232 if (regionA == null || regionB == null) {
233 LOG.warn("Unexpected references regionA="
234 + (regionA == null ? "null" : regionA.getRegionNameAsString())
235 + ",regionB="
236 + (regionB == null ? "null" : regionB.getRegionNameAsString())
237 + " in merged region " + e.getKey().getRegionNameAsString());
238 } else {
239 if (cleanMergeRegion(e.getKey(), regionA, regionB)) {
240 mergeCleaned++;
241 }
242 }
243 }
244
245
246
247 Map<HRegionInfo, Result> splitParents = scanTriple.getThird();
248
249
250 int splitCleaned = 0;
251
252 HashSet<String> parentNotCleaned = new HashSet<String>();
253 for (Map.Entry<HRegionInfo, Result> e : splitParents.entrySet()) {
254 if (!parentNotCleaned.contains(e.getKey().getEncodedName()) &&
255 cleanParent(e.getKey(), e.getValue())) {
256 splitCleaned++;
257 } else {
258
259 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(e.getValue());
260 parentNotCleaned.add(daughters.getFirst().getEncodedName());
261 parentNotCleaned.add(daughters.getSecond().getEncodedName());
262 }
263 }
264 if ((mergeCleaned + splitCleaned) != 0) {
265 LOG.info("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
266 + " unreferenced merged region(s) and " + splitCleaned
267 + " unreferenced parent region(s)");
268 } else if (LOG.isTraceEnabled()) {
269 LOG.trace("Scanned " + count + " catalog row(s), gc'd " + mergeCleaned
270 + " unreferenced merged region(s) and " + splitCleaned
271 + " unreferenced parent region(s)");
272 }
273 return mergeCleaned + splitCleaned;
274 } finally {
275 alreadyRunning.set(false);
276 }
277 }
278
279
280
281
282
283 static class SplitParentFirstComparator implements Comparator<HRegionInfo> {
284 Comparator<byte[]> rowEndKeyComparator = new Bytes.RowEndKeyComparator();
285 @Override
286 public int compare(HRegionInfo left, HRegionInfo right) {
287
288
289 if (left == null) return -1;
290 if (right == null) return 1;
291
292 int result = left.getTable().compareTo(right.getTable());
293 if (result != 0) return result;
294
295 result = Bytes.compareTo(left.getStartKey(), right.getStartKey());
296 if (result != 0) return result;
297
298 result = rowEndKeyComparator.compare(right.getEndKey(), left.getEndKey());
299
300 return result;
301 }
302 }
303
304
305
306
307
308
309
310
311
312
313 boolean cleanParent(final HRegionInfo parent, Result rowContent)
314 throws IOException {
315 boolean result = false;
316
317
318
319 if (rowContent.getValue(HConstants.CATALOG_FAMILY,
320 HConstants.MERGEA_QUALIFIER) != null) {
321
322 return result;
323 }
324
325 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(rowContent);
326 Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
327 Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
328 if (hasNoReferences(a) && hasNoReferences(b)) {
329 LOG.debug("Deleting region " + parent.getRegionNameAsString() +
330 " because daughter splits no longer hold references");
331 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
332 if (LOG.isTraceEnabled()) LOG.trace("Archiving parent region: " + parent);
333 HFileArchiver.archiveRegion(this.services.getConfiguration(), fs, parent);
334 MetaEditor.deleteRegion(this.server.getCatalogTracker(), parent);
335 result = true;
336 }
337 return result;
338 }
339
340
341
342
343
344
345
346 private boolean hasNoReferences(final Pair<Boolean, Boolean> p) {
347 return !p.getFirst() || !p.getSecond();
348 }
349
350
351
352
353
354
355
356
357
358
359
360 Pair<Boolean, Boolean> checkDaughterInFs(final HRegionInfo parent, final HRegionInfo daughter)
361 throws IOException {
362 if (daughter == null) {
363 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
364 }
365
366 FileSystem fs = this.services.getMasterFileSystem().getFileSystem();
367 Path rootdir = this.services.getMasterFileSystem().getRootDir();
368 Path tabledir = FSUtils.getTableDir(rootdir, daughter.getTable());
369
370 Path daughterRegionDir = new Path(tabledir, daughter.getEncodedName());
371
372 HRegionFileSystem regionFs = null;
373
374 try {
375 if (!FSUtils.isExists(fs, daughterRegionDir)) {
376 return new Pair<Boolean, Boolean>(Boolean.FALSE, Boolean.FALSE);
377 }
378 } catch (IOException ioe) {
379 LOG.warn("Error trying to determine if daughter region exists, " +
380 "assuming exists and has references", ioe);
381 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
382 }
383
384 try {
385 regionFs = HRegionFileSystem.openRegionFromFileSystem(
386 this.services.getConfiguration(), fs, tabledir, daughter, true);
387 } catch (IOException e) {
388 LOG.warn("Error trying to determine referenced files from : " + daughter.getEncodedName()
389 + ", to: " + parent.getEncodedName() + " assuming has references", e);
390 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.TRUE);
391 }
392
393 boolean references = false;
394 HTableDescriptor parentDescriptor = getTableDescriptor(parent.getTable());
395 for (HColumnDescriptor family: parentDescriptor.getFamilies()) {
396 if ((references = regionFs.hasReferences(family.getNameAsString()))) {
397 break;
398 }
399 }
400 return new Pair<Boolean, Boolean>(Boolean.TRUE, Boolean.valueOf(references));
401 }
402
403 private HTableDescriptor getTableDescriptor(final TableName tableName)
404 throws FileNotFoundException, IOException {
405 return this.services.getTableDescriptors().get(tableName);
406 }
407
408
409
410
411
412
413
414
415 public boolean cleanMergeQualifier(final HRegionInfo region)
416 throws IOException {
417
418
419 Pair<HRegionInfo, HRegionInfo> mergeRegions = MetaReader
420 .getRegionsFromMergeQualifier(this.services.getCatalogTracker(),
421 region.getRegionName());
422 if (mergeRegions == null
423 || (mergeRegions.getFirst() == null && mergeRegions.getSecond() == null)) {
424
425 return true;
426 }
427
428 if (mergeRegions.getFirst() == null || mergeRegions.getSecond() == null) {
429 LOG.error("Merged region " + region.getRegionNameAsString()
430 + " has only one merge qualifier in META.");
431 return false;
432 }
433 return cleanMergeRegion(region, mergeRegions.getFirst(),
434 mergeRegions.getSecond());
435 }
436 }