1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.regex.Matcher;
24 import java.util.regex.Pattern;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
34 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
37 import org.apache.hadoop.hbase.io.Reference;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54
55
56 private static final Pattern HFILE_NAME_PATTERN =
57 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58
59
60
61
62
63
64
65
66 private static final Pattern REF_NAME_PATTERN =
67 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69
70
71 private Configuration conf;
72
73
74 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
75
76
77 private final Reference reference;
78
79
80 private final HFileLink link;
81
82
83 private final FileStatus fileStatus;
84
85 private RegionCoprocessorHost coprocessorHost;
86
87
88
89
90
91
92
93 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
94 throws IOException {
95 this(conf, fs, fs.getFileStatus(path));
96 }
97
98
99
100
101
102
103
104 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
105 throws IOException {
106 this.conf = conf;
107 this.fileStatus = fileStatus;
108 Path p = fileStatus.getPath();
109 if (HFileLink.isHFileLink(p)) {
110
111 this.reference = null;
112 this.link = new HFileLink(conf, p);
113 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
114 } else if (isReference(p)) {
115 this.reference = Reference.read(fs, p);
116 Path referencePath = getReferredToFile(p);
117 if (HFileLink.isHFileLink(referencePath)) {
118
119 this.link = new HFileLink(conf, referencePath);
120 } else {
121
122 this.link = null;
123 }
124 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
125 " reference to " + referencePath);
126 } else if (isHFile(p)) {
127
128 this.reference = null;
129 this.link = null;
130 } else {
131 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
132 }
133 }
134
135
136
137
138
139 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
140 this.coprocessorHost = coprocessorHost;
141 }
142
143
144
145
146
147 public Reference getReference() {
148 return this.reference;
149 }
150
151
152 public boolean isReference() {
153 return this.reference != null;
154 }
155
156
157 public boolean isTopReference() {
158 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
159 }
160
161
162 public boolean isLink() {
163 return this.link != null && this.reference == null;
164 }
165
166
167 public HDFSBlocksDistribution getHDFSBlockDistribution() {
168 return this.hdfsBlocksDistribution;
169 }
170
171
172
173
174
175
176
177 public StoreFile.Reader open(final FileSystem fs,
178 final CacheConfig cacheConf) throws IOException {
179 FSDataInputStreamWrapper in;
180 FileStatus status;
181
182 if (this.link != null) {
183
184 in = new FSDataInputStreamWrapper(fs, this.link);
185 status = this.link.getFileStatus(fs);
186 } else if (this.reference != null) {
187
188 Path referencePath = getReferredToFile(this.getPath());
189 in = new FSDataInputStreamWrapper(fs, referencePath);
190 status = fs.getFileStatus(referencePath);
191 } else {
192 in = new FSDataInputStreamWrapper(fs, this.getPath());
193 status = fileStatus;
194 }
195 long length = status.getLen();
196 if (this.reference != null) {
197 hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);
198 } else {
199 hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
200 }
201 StoreFile.Reader reader = null;
202 if (this.coprocessorHost != null) {
203 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
204 cacheConf, reference);
205 }
206 if (reader == null) {
207 if (this.reference != null) {
208 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
209 conf);
210 } else {
211 reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
212 }
213 }
214 if (this.coprocessorHost != null) {
215 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
216 cacheConf, reference, reader);
217 }
218 return reader;
219 }
220
221
222
223
224 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
225 throws IOException {
226 FileStatus status = getReferencedFileStatus(fs);
227 if (this.reference != null) {
228 return computeRefFileHDFSBlockDistribution(fs, reference, status);
229 } else {
230 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
231 }
232 }
233
234
235
236
237
238
239 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
240 FileStatus status;
241 if (this.reference != null) {
242 if (this.link != null) {
243
244 status = link.getFileStatus(fs);
245 } else {
246
247 Path referencePath = getReferredToFile(this.getPath());
248 status = fs.getFileStatus(referencePath);
249 }
250 } else {
251 if (this.link != null) {
252
253 status = link.getFileStatus(fs);
254 } else {
255 status = this.fileStatus;
256 }
257 }
258 return status;
259 }
260
261
262 public Path getPath() {
263 return this.fileStatus.getPath();
264 }
265
266
267 public FileStatus getFileStatus() {
268 return this.fileStatus;
269 }
270
271
272 public long getModificationTime() {
273 return this.fileStatus.getModificationTime();
274 }
275
276 @Override
277 public String toString() {
278 return this.getPath() +
279 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
280 }
281
282
283
284
285
286 public static boolean isHFile(final Path path) {
287 return isHFile(path.getName());
288 }
289
290 public static boolean isHFile(final String fileName) {
291 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
292 return m.matches() && m.groupCount() > 0;
293 }
294
295
296
297
298
299 public static boolean isReference(final Path path) {
300 return isReference(path.getName());
301 }
302
303
304
305
306
307 public static boolean isReference(final String name) {
308 Matcher m = REF_NAME_PATTERN.matcher(name);
309 return m.matches() && m.groupCount() > 1;
310 }
311
312
313
314
315
316
317
318
319 public static Path getReferredToFile(final Path p) {
320 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
321 if (m == null || !m.matches()) {
322 LOG.warn("Failed match of store file name " + p.toString());
323 throw new IllegalArgumentException("Failed match of store file name " +
324 p.toString());
325 }
326
327
328 String otherRegion = m.group(2);
329
330 Path tableDir = p.getParent().getParent().getParent();
331 String nameStrippedOfSuffix = m.group(1);
332 LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
333
334
335
336 return new Path(new Path(new Path(tableDir, otherRegion),
337 p.getParent().getName()), nameStrippedOfSuffix);
338 }
339
340
341
342
343
344
345 public static boolean validateStoreFileName(final String fileName) {
346 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
347 return(true);
348 return !fileName.contains("-");
349 }
350
351
352
353
354
355
356 public static boolean isValid(final FileStatus fileStatus)
357 throws IOException {
358 final Path p = fileStatus.getPath();
359
360 if (fileStatus.isDir())
361 return false;
362
363
364
365
366 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
367 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
368 return false;
369 }
370
371 return validateStoreFileName(p.getName());
372 }
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
388 final FileSystem fs, final Reference reference, final FileStatus status)
389 throws IOException {
390 if (status == null) {
391 return null;
392 }
393
394 long start = 0;
395 long length = 0;
396
397 if (Reference.isTopFileRegion(reference.getFileRegion())) {
398 start = status.getLen()/2;
399 length = status.getLen() - status.getLen()/2;
400 } else {
401 start = 0;
402 length = status.getLen()/2;
403 }
404 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
405 }
406 }