View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.IOException;
22  import java.io.InterruptedIOException;
23  import java.util.ArrayList;
24  import java.util.HashSet;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.Set;
28  import java.util.concurrent.locks.Lock;
29  import java.util.concurrent.locks.ReentrantLock;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileStatus;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.fs.PathFilter;
39  import org.apache.hadoop.hbase.ClusterId;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.InvalidFamilyOperationException;
46  import org.apache.hadoop.hbase.RemoteExceptionHandler;
47  import org.apache.hadoop.hbase.Server;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.backup.HFileArchiver;
50  import org.apache.hadoop.hbase.catalog.MetaReader;
51  import org.apache.hadoop.hbase.client.Result;
52  import org.apache.hadoop.hbase.exceptions.DeserializationException;
53  import org.apache.hadoop.hbase.fs.HFileSystem;
54  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
55  import org.apache.hadoop.hbase.regionserver.HRegion;
56  import org.apache.hadoop.hbase.regionserver.wal.HLog;
57  import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
58  import org.apache.hadoop.hbase.util.Bytes;
59  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
60  import org.apache.hadoop.hbase.util.FSTableDescriptors;
61  import org.apache.hadoop.hbase.util.FSUtils;
62  import org.apache.zookeeper.KeeperException;
63  
64  /**
65   * This class abstracts a bunch of operations the HMaster needs to interact with
66   * the underlying file system, including splitting log files, checking file
67   * system status, etc.
68   */
69  @InterfaceAudience.Private
70  public class MasterFileSystem {
71    private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
72    // HBase configuration
73    Configuration conf;
74    // master status
75    Server master;
76    // metrics for master
77    private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
78    // Persisted unique cluster ID
79    private ClusterId clusterId;
80    // Keep around for convenience.
81    private final FileSystem fs;
82    // Is the fileystem ok?
83    private volatile boolean fsOk = true;
84    // The Path to the old logs dir
85    private final Path oldLogDir;
86    // root hbase directory on the FS
87    private final Path rootdir;
88    // hbase temp directory used for table construction and deletion
89    private final Path tempdir;
90    // create the split log lock
91    final Lock splitLogLock = new ReentrantLock();
92    final boolean distributedLogReplay;
93    final SplitLogManager splitLogManager;
94    private final MasterServices services;
95  
96    final static PathFilter META_FILTER = new PathFilter() {
97      @Override
98      public boolean accept(Path p) {
99        return HLogUtil.isMetaFile(p);
100     }
101   };
102 
103   final static PathFilter NON_META_FILTER = new PathFilter() {
104     @Override
105     public boolean accept(Path p) {
106       return !HLogUtil.isMetaFile(p);
107     }
108   };
109 
110   public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
111   throws IOException {
112     this.conf = master.getConfiguration();
113     this.master = master;
114     this.services = services;
115     // Set filesystem to be that of this.rootdir else we get complaints about
116     // mismatched filesystems if hbase.rootdir is hdfs and fs.defaultFS is
117     // default localfs.  Presumption is that rootdir is fully-qualified before
118     // we get to here with appropriate fs scheme.
119     this.rootdir = FSUtils.getRootDir(conf);
120     this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
121     // Cover both bases, the old way of setting default fs and the new.
122     // We're supposed to run on 0.20 and 0.21 anyways.
123     this.fs = this.rootdir.getFileSystem(conf);
124     FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
125     // make sure the fs has the same conf
126     fs.setConf(conf);
127     // setup the filesystem variable
128     // set up the archived logs path
129     this.oldLogDir = createInitialFileSystemLayout();
130     HFileSystem.addLocationsOrderInterceptor(conf);
131     try {
132       this.splitLogManager = new SplitLogManager(master.getZooKeeper(), master.getConfiguration(),
133  master, services,
134               master.getServerName(), masterRecovery);
135     } catch (KeeperException e) {
136       throw new IOException(e);
137     }
138     this.distributedLogReplay = (this.splitLogManager.getRecoveryMode() == RecoveryMode.LOG_REPLAY);
139   }
140 
141   /**
142    * Create initial layout in filesystem.
143    * <ol>
144    * <li>Check if the meta region exists and is readable, if not create it.
145    * Create hbase.version and the hbase:meta directory if not one.
146    * </li>
147    * <li>Create a log archive directory for RS to put archived logs</li>
148    * </ol>
149    * Idempotent.
150    */
151   private Path createInitialFileSystemLayout() throws IOException {
152     // check if the root directory exists
153     checkRootDir(this.rootdir, conf, this.fs);
154 
155     // check if temp directory exists and clean it
156     checkTempDir(this.tempdir, conf, this.fs);
157 
158     Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
159 
160     // Make sure the region servers can archive their old logs
161     if(!this.fs.exists(oldLogDir)) {
162       this.fs.mkdirs(oldLogDir);
163     }
164 
165     return oldLogDir;
166   }
167 
168   public FileSystem getFileSystem() {
169     return this.fs;
170   }
171 
172   /**
173    * Get the directory where old logs go
174    * @return the dir
175    */
176   public Path getOldLogDir() {
177     return this.oldLogDir;
178   }
179 
180   /**
181    * Checks to see if the file system is still accessible.
182    * If not, sets closed
183    * @return false if file system is not available
184    */
185   public boolean checkFileSystem() {
186     if (this.fsOk) {
187       try {
188         FSUtils.checkFileSystemAvailable(this.fs);
189         FSUtils.checkDfsSafeMode(this.conf);
190       } catch (IOException e) {
191         master.abort("Shutting down HBase cluster: file system not available", e);
192         this.fsOk = false;
193       }
194     }
195     return this.fsOk;
196   }
197 
198   /**
199    * @return HBase root dir.
200    */
201   public Path getRootDir() {
202     return this.rootdir;
203   }
204 
205   /**
206    * @return HBase temp dir.
207    */
208   public Path getTempDir() {
209     return this.tempdir;
210   }
211 
212   /**
213    * @return The unique identifier generated for this cluster
214    */
215   public ClusterId getClusterId() {
216     return clusterId;
217   }
218 
219   /**
220    * Inspect the log directory to find dead servers which need recovery work
221    * @return A set of ServerNames which aren't running but still have WAL files left in file system
222    */
223   Set<ServerName> getFailedServersFromLogFolders() {
224     boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
225       HLog.SPLIT_SKIP_ERRORS_DEFAULT);
226 
227     Set<ServerName> serverNames = new HashSet<ServerName>();
228     Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
229 
230     do {
231       if (master.isStopped()) {
232         LOG.warn("Master stopped while trying to get failed servers.");
233         break;
234       }
235       try {
236         if (!this.fs.exists(logsDirPath)) return serverNames;
237         FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
238         // Get online servers after getting log folders to avoid log folder deletion of newly
239         // checked in region servers . see HBASE-5916
240         Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
241             .keySet();
242 
243         if (logFolders == null || logFolders.length == 0) {
244           LOG.debug("No log files to split, proceeding...");
245           return serverNames;
246         }
247         for (FileStatus status : logFolders) {
248           String sn = status.getPath().getName();
249           // truncate splitting suffix if present (for ServerName parsing)
250           if (sn.endsWith(HLog.SPLITTING_EXT)) {
251             sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
252           }
253           ServerName serverName = ServerName.parseServerName(sn);
254           if (!onlineServers.contains(serverName)) {
255             LOG.info("Log folder " + status.getPath() + " doesn't belong "
256                 + "to a known region server, splitting");
257             serverNames.add(serverName);
258           } else {
259             LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
260           }
261         }
262         retrySplitting = false;
263       } catch (IOException ioe) {
264         LOG.warn("Failed getting failed servers to be recovered.", ioe);
265         if (!checkFileSystem()) {
266           LOG.warn("Bad Filesystem, exiting");
267           Runtime.getRuntime().halt(1);
268         }
269         try {
270           if (retrySplitting) {
271             Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
272           }
273         } catch (InterruptedException e) {
274           LOG.warn("Interrupted, aborting since cannot return w/o splitting");
275           Thread.currentThread().interrupt();
276           retrySplitting = false;
277           Runtime.getRuntime().halt(1);
278         }
279       }
280     } while (retrySplitting);
281 
282     return serverNames;
283   }
284 
285   public void splitLog(final ServerName serverName) throws IOException {
286     Set<ServerName> serverNames = new HashSet<ServerName>();
287     serverNames.add(serverName);
288     splitLog(serverNames);
289   }
290 
291   /**
292    * Specialized method to handle the splitting for meta HLog
293    * @param serverName
294    * @throws IOException
295    */
296   public void splitMetaLog(final ServerName serverName) throws IOException {
297     Set<ServerName> serverNames = new HashSet<ServerName>();
298     serverNames.add(serverName);
299     splitMetaLog(serverNames);
300   }
301 
302   /**
303    * Specialized method to handle the splitting for meta HLog
304    * @param serverNames
305    * @throws IOException
306    */
307   public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
308     splitLog(serverNames, META_FILTER);
309   }
310 
311   private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
312     List<Path> logDirs = new ArrayList<Path>();
313     boolean needReleaseLock = false;
314     if (!this.services.isInitialized()) {
315       // during master initialization, we could have multiple places splitting a same wal
316       this.splitLogLock.lock();
317       needReleaseLock = true;
318     }
319     try {
320       for (ServerName serverName : serverNames) {
321         Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
322         Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
323         // Rename the directory so a rogue RS doesn't create more HLogs
324         if (fs.exists(logDir)) {
325           if (!this.fs.rename(logDir, splitDir)) {
326             throw new IOException("Failed fs.rename for log split: " + logDir);
327           }
328           logDir = splitDir;
329           LOG.debug("Renamed region directory: " + splitDir);
330         } else if (!fs.exists(splitDir)) {
331           LOG.info("Log dir for server " + serverName + " does not exist");
332           continue;
333         }
334         logDirs.add(splitDir);
335       }
336     } finally {
337       if (needReleaseLock) {
338         this.splitLogLock.unlock();
339       }
340     }
341     return logDirs;
342   }
343 
344   /**
345    * Mark regions in recovering state when distributedLogReplay are set true
346    * @param serverNames Set of ServerNames to be replayed wals in order to recover changes contained
347    *          in them
348    * @throws IOException
349    */
350   public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
351     if (!this.distributedLogReplay) {
352       return;
353     }
354     // mark regions in recovering state
355     for (ServerName serverName : serverNames) {
356       NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
357       if (regions == null) {
358         continue;
359       }
360       try {
361         this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
362       } catch (KeeperException e) {
363         throw new IOException(e);
364       }
365     }
366   }
367 
368   /**
369    * Mark regions in recovering state when distributedLogReplay are set true
370    * @param serverName Failed region server whose wals to be replayed
371    * @param regions Set of regions to be recovered
372    * @throws IOException
373    */
374   public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
375     if (!this.distributedLogReplay) {
376       return;
377     }
378     // mark regions in recovering state
379     if (regions == null || regions.isEmpty()) {
380       return;
381     }
382     try {
383       this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
384     } catch (KeeperException e) {
385       throw new IOException(e);
386     }
387   }
388 
389   public void splitLog(final Set<ServerName> serverNames) throws IOException {
390     splitLog(serverNames, NON_META_FILTER);
391   }
392 
393   /**
394    * Wrapper function on {@link SplitLogManager#removeStaleRecoveringRegionsFromZK(Set)}
395    * @param failedServers
396    * @throws KeeperException
397    */
398   void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
399       throws KeeperException {
400     this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
401   }
402 
403   /**
404    * This method is the base split method that splits HLog files matching a filter. Callers should
405    * pass the appropriate filter for meta and non-meta HLogs.
406    * @param serverNames
407    * @param filter
408    * @throws IOException
409    */
410   public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
411     long splitTime = 0, splitLogSize = 0;
412     List<Path> logDirs = getLogDirs(serverNames);
413 
414     splitLogManager.handleDeadWorkers(serverNames);
415     splitTime = EnvironmentEdgeManager.currentTimeMillis();
416     splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
417     splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
418 
419     if (this.metricsMasterFilesystem != null) {
420       if (filter == META_FILTER) {
421         this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
422       } else {
423         this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
424       }
425     }
426   }
427 
428   /**
429    * Get the rootdir.  Make sure its wholesome and exists before returning.
430    * @param rd
431    * @param c
432    * @param fs
433    * @return hbase.rootdir (after checks for existence and bootstrapping if
434    * needed populating the directory with necessary bootup files).
435    * @throws IOException
436    */
437   @SuppressWarnings("deprecation")
438   private Path checkRootDir(final Path rd, final Configuration c,
439     final FileSystem fs)
440   throws IOException {
441     // If FS is in safe mode wait till out of it.
442     FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
443     // Filesystem is good. Go ahead and check for hbase.rootdir.
444     try {
445       if (!fs.exists(rd)) {
446         fs.mkdirs(rd);
447         // DFS leaves safe mode with 0 DNs when there are 0 blocks.
448         // We used to handle this by checking the current DN count and waiting until
449         // it is nonzero. With security, the check for datanode count doesn't work --
450         // it is a privileged op. So instead we adopt the strategy of the jobtracker
451         // and simply retry file creation during bootstrap indefinitely. As soon as
452         // there is one datanode it will succeed. Permission problems should have
453         // already been caught by mkdirs above.
454         FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
455           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
456             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
457       } else {
458         if (!fs.isDirectory(rd)) {
459           throw new IllegalArgumentException(rd.toString() + " is not a directory");
460         }
461         // as above
462         FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
463           10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
464             HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
465       }
466     } catch (DeserializationException de) {
467       LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
468       IOException ioe = new IOException();
469       ioe.initCause(de);
470       throw ioe;
471     } catch (IllegalArgumentException iae) {
472       LOG.fatal("Please fix invalid configuration for "
473         + HConstants.HBASE_DIR + " " + rd.toString(), iae);
474       throw iae;
475     }
476     // Make sure cluster ID exists
477     if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
478         HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
479       FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
480     }
481     clusterId = FSUtils.getClusterId(fs, rd);
482 
483     // Make sure the meta region directory exists!
484     if (!FSUtils.metaRegionExists(fs, rd)) {
485       bootstrap(rd, c);
486     } else {
487       // Migrate table descriptor files if necessary
488       org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
489         .migrateFSTableDescriptorsIfNecessary(fs, rd);
490     }
491 
492     // Create tableinfo-s for hbase:meta if not already there.
493 
494     // meta table is a system table, so descriptors are predefined,
495     // we should get them from registry.
496     FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
497     fsd.createTableDescriptor(
498       new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
499 
500     return rd;
501   }
502 
503   /**
504    * Make sure the hbase temp directory exists and is empty.
505    * NOTE that this method is only executed once just after the master becomes the active one.
506    */
507   private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
508       throws IOException {
509     // If the temp directory exists, clear the content (left over, from the previous run)
510     if (fs.exists(tmpdir)) {
511       // Archive table in temp, maybe left over from failed deletion,
512       // if not the cleaner will take care of them.
513       for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
514         for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
515           HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
516         }
517       }
518       if (!fs.delete(tmpdir, true)) {
519         throw new IOException("Unable to clean the temp directory: " + tmpdir);
520       }
521     }
522 
523     // Create the temp directory
524     if (!fs.mkdirs(tmpdir)) {
525       throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
526     }
527   }
528 
529   private static void bootstrap(final Path rd, final Configuration c)
530   throws IOException {
531     LOG.info("BOOTSTRAP: creating hbase:meta region");
532     try {
533       // Bootstrapping, make sure blockcache is off.  Else, one will be
534       // created here in bootstrap and it'll need to be cleaned up.  Better to
535       // not make it in first place.  Turn off block caching for bootstrap.
536       // Enable after.
537       HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
538       HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
539       setInfoFamilyCachingForMeta(metaDescriptor, false);
540       HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
541       setInfoFamilyCachingForMeta(metaDescriptor, true);
542       HRegion.closeHRegion(meta);
543     } catch (IOException e) {
544       e = RemoteExceptionHandler.checkIOException(e);
545       LOG.error("bootstrap", e);
546       throw e;
547     }
548   }
549 
550   /**
551    * Enable in memory caching for hbase:meta
552    */
553   public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
554       final boolean b) {
555     for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
556       if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
557         hcd.setBlockCacheEnabled(b);
558         hcd.setInMemory(b);
559       }
560     }
561   }
562 
563   public void deleteRegion(HRegionInfo region) throws IOException {
564     HFileArchiver.archiveRegion(conf, fs, region);
565   }
566 
567   public void deleteTable(TableName tableName) throws IOException {
568     fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
569   }
570 
571   /**
572    * Move the specified table to the hbase temp directory
573    * @param tableName Table name to move
574    * @return The temp location of the table moved
575    * @throws IOException in case of file-system failure
576    */
577   public Path moveTableToTemp(TableName tableName) throws IOException {
578     Path srcPath = FSUtils.getTableDir(rootdir, tableName);
579     Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
580 
581     // Ensure temp exists
582     if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
583       throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
584     }
585 
586     if (!fs.rename(srcPath, tempPath)) {
587       throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
588     }
589 
590     return tempPath;
591   }
592 
593   public void updateRegionInfo(HRegionInfo region) {
594     // TODO implement this.  i think this is currently broken in trunk i don't
595     //      see this getting updated.
596     //      @see HRegion.checkRegioninfoOnFilesystem()
597   }
598 
599   public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
600       throws IOException {
601     // archive family store files
602     Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
603     HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
604 
605     // delete the family folder
606     Path familyDir = new Path(tableDir,
607       new Path(region.getEncodedName(), Bytes.toString(familyName)));
608     if (fs.delete(familyDir, true) == false) {
609       throw new IOException("Could not delete family "
610           + Bytes.toString(familyName) + " from FileSystem for region "
611           + region.getRegionNameAsString() + "(" + region.getEncodedName()
612           + ")");
613     }
614   }
615 
616   public void stop() {
617     if (splitLogManager != null) {
618       this.splitLogManager.stop();
619     }
620   }
621 
622   /**
623    * Delete column of a table
624    * @param tableName
625    * @param familyName
626    * @return Modified HTableDescriptor with requested column deleted.
627    * @throws IOException
628    */
629   public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
630       throws IOException {
631     LOG.info("DeleteColumn. Table = " + tableName
632         + " family = " + Bytes.toString(familyName));
633     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
634     htd.removeFamily(familyName);
635     this.services.getTableDescriptors().add(htd);
636     return htd;
637   }
638 
639   /**
640    * Modify Column of a table
641    * @param tableName
642    * @param hcd HColumnDesciptor
643    * @return Modified HTableDescriptor with the column modified.
644    * @throws IOException
645    */
646   public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
647       throws IOException {
648     LOG.info("AddModifyColumn. Table = " + tableName
649         + " HCD = " + hcd.toString());
650 
651     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
652     byte [] familyName = hcd.getName();
653     if(!htd.hasFamily(familyName)) {
654       throw new InvalidFamilyOperationException("Family '" +
655         Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
656     }
657     htd.addFamily(hcd);
658     this.services.getTableDescriptors().add(htd);
659     return htd;
660   }
661 
662   /**
663    * Add column to a table
664    * @param tableName
665    * @param hcd
666    * @return Modified HTableDescriptor with new column added.
667    * @throws IOException
668    */
669   public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
670       throws IOException {
671     LOG.info("AddColumn. Table = " + tableName + " HCD = " +
672       hcd.toString());
673     HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
674     if (htd == null) {
675       throw new InvalidFamilyOperationException("Family '" +
676         hcd.getNameAsString() + "' cannot be modified as HTD is null");
677     }
678     htd.addFamily(hcd);
679     this.services.getTableDescriptors().add(htd);
680     return htd;
681   }
682 
683   private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
684       throws IOException {
685     if (!this.master.isStopped()) {
686       try {
687         this.master.getCatalogTracker().waitForMeta();
688         return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
689       } catch (InterruptedException e) {
690         throw (InterruptedIOException)new InterruptedIOException().initCause(e);
691       }
692     }
693     return null;
694   }
695 
696   /**
697    * The function is used in SSH to set recovery mode based on configuration after all outstanding
698    * log split tasks drained.
699    * @throws KeeperException
700    * @throws InterruptedIOException
701    */
702   public void setLogRecoveryMode() throws IOException {
703     try {
704       this.splitLogManager.setRecoveryMode(false);
705     } catch (KeeperException e) {
706       throw new IOException(e);
707     }
708   }
709 
710   public RecoveryMode getLogRecoveryMode() {
711     return this.splitLogManager.getRecoveryMode();
712   }
713 }