1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.NavigableSet;
25 import java.util.TreeSet;
26 import java.util.UUID;
27 import java.util.concurrent.atomic.AtomicLong;
28 import java.util.regex.Matcher;
29 import java.util.regex.Pattern;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.fs.PathFilter;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.ServerName;
44 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor;
45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
46 import org.apache.hadoop.hbase.util.FSUtils;
47
48 import com.google.protobuf.TextFormat;
49
50 @InterfaceAudience.Private
51 public class HLogUtil {
52 static final Log LOG = LogFactory.getLog(HLogUtil.class);
53
54
55
56
57 private static final Pattern pattern =
58 Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
59
60
61
62
63
64
65
66 public static boolean validateHLogFilename(String filename) {
67 return pattern.matcher(filename).matches();
68 }
69
70
71
72
73
74
75
76
77
78
79
80 public static String getHLogDirectoryName(final String serverName) {
81 StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
82 dirName.append("/");
83 dirName.append(serverName);
84 return dirName.toString();
85 }
86
87
88
89
90
91
92
93 public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
94 return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
95 }
96
97
98
99
100
101
102
103
104
105
106 public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
107 throws IOException {
108 Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
109 + System.currentTimeMillis());
110 if (!fs.rename(edits, moveAsideName)) {
111 LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
112 }
113 return moveAsideName;
114 }
115
116
117
118
119
120
121
122
123
124 public static ServerName getServerNameFromHLogDirectoryName(
125 Configuration conf, String path) throws IOException {
126 if (path == null
127 || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
128 return null;
129 }
130
131 if (conf == null) {
132 throw new IllegalArgumentException("parameter conf must be set");
133 }
134
135 final String rootDir = conf.get(HConstants.HBASE_DIR);
136 if (rootDir == null || rootDir.isEmpty()) {
137 throw new IllegalArgumentException(HConstants.HBASE_DIR
138 + " key not found in conf.");
139 }
140
141 final StringBuilder startPathSB = new StringBuilder(rootDir);
142 if (!rootDir.endsWith("/"))
143 startPathSB.append('/');
144 startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
145 if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
146 startPathSB.append('/');
147 final String startPath = startPathSB.toString();
148
149 String fullPath;
150 try {
151 fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
152 } catch (IllegalArgumentException e) {
153 LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
154 return null;
155 }
156
157 if (!fullPath.startsWith(startPath)) {
158 return null;
159 }
160
161 final String serverNameAndFile = fullPath.substring(startPath.length());
162
163 if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
164
165 return null;
166 }
167
168 Path p = new Path(path);
169 return getServerNameFromHLogDirectoryName(p);
170 }
171
172
173
174
175
176
177
178
179 public static ServerName getServerNameFromHLogDirectoryName(Path logFile) {
180 Path logDir = logFile.getParent();
181 String logDirName = logDir.getName();
182 if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
183 logDir = logFile;
184 logDirName = logDir.getName();
185 }
186 ServerName serverName = null;
187 if (logDirName.endsWith(HLog.SPLITTING_EXT)) {
188 logDirName = logDirName.substring(0, logDirName.length() - HLog.SPLITTING_EXT.length());
189 }
190 try {
191 serverName = ServerName.parseServerName(logDirName);
192 } catch (IllegalArgumentException ex) {
193 serverName = null;
194 LOG.warn("Invalid log file path=" + logFile, ex);
195 }
196 if (serverName != null && serverName.getStartcode() < 0) {
197 LOG.warn("Invalid log file path=" + logFile);
198 return null;
199 }
200 return serverName;
201 }
202
203
204
205
206
207
208
209
210
211
212 public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
213 final Path regiondir) throws IOException {
214 NavigableSet<Path> filesSorted = new TreeSet<Path>();
215 Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
216 if (!fs.exists(editsdir))
217 return filesSorted;
218 FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
219 @Override
220 public boolean accept(Path p) {
221 boolean result = false;
222 try {
223
224
225
226
227 Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
228 result = fs.isFile(p) && m.matches();
229
230
231 if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
232 result = false;
233 }
234 } catch (IOException e) {
235 LOG.warn("Failed isFile check on " + p);
236 }
237 return result;
238 }
239 });
240 if (files == null)
241 return filesSorted;
242 for (FileStatus status : files) {
243 filesSorted.add(status.getPath());
244 }
245 return filesSorted;
246 }
247
248 public static boolean isMetaFile(Path p) {
249 return isMetaFile(p.getName());
250 }
251
252 public static boolean isMetaFile(String p) {
253 if (p != null && p.endsWith(HLog.META_HLOG_FILE_EXTN)) {
254 return true;
255 }
256 return false;
257 }
258
259
260
261
262
263
264
265
266 public static void writeCompactionMarker(HLog log, HTableDescriptor htd, HRegionInfo info,
267 final CompactionDescriptor c, AtomicLong sequenceId) throws IOException {
268 WALEdit e = WALEdit.createCompaction(info, c);
269 long now = EnvironmentEdgeManager.currentTimeMillis();
270 TableName tn = TableName.valueOf(c.getTableName().toByteArray());
271 long txid = log.appendNoSync(info, tn, e, new ArrayList<UUID>(), now, htd, sequenceId,
272 false, HConstants.NO_NONCE, HConstants.NO_NONCE);
273 log.sync(txid);
274
275 if (LOG.isTraceEnabled()) {
276 LOG.trace("Appended compaction marker " + TextFormat.shortDebugString(c));
277 }
278 }
279 }