1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collections;
24 import java.util.HashMap;
25 import java.util.HashSet;
26 import java.util.Iterator;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.concurrent.ThreadPoolExecutor;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.classification.InterfaceStability;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
43 import org.apache.hadoop.hbase.HConstants;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.Stoppable;
46 import org.apache.hadoop.hbase.catalog.MetaReader;
47 import org.apache.hadoop.hbase.errorhandling.ForeignException;
48 import org.apache.hadoop.hbase.executor.ExecutorService;
49 import org.apache.hadoop.hbase.master.AssignmentManager;
50 import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
51 import org.apache.hadoop.hbase.master.MasterFileSystem;
52 import org.apache.hadoop.hbase.master.MasterServices;
53 import org.apache.hadoop.hbase.master.MetricsMaster;
54 import org.apache.hadoop.hbase.master.SnapshotSentinel;
55 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
56 import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner;
57 import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
58 import org.apache.hadoop.hbase.procedure.Procedure;
59 import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
60 import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
61 import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinatorRpcs;
62 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
63 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
64 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
65 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription.Type;
66 import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
67 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
68 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
69 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper;
70 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
71 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
72 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
73 import org.apache.hadoop.hbase.snapshot.SnapshotExistsException;
74 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
75 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
76 import org.apache.hadoop.hbase.snapshot.TablePartiallyOpenException;
77 import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
78 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
79 import org.apache.hadoop.hbase.util.FSUtils;
80 import org.apache.zookeeper.KeeperException;
81
82
83
84
85
86
87
88
89
90
91 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
92 @InterfaceStability.Unstable
93 public class SnapshotManager extends MasterProcedureManager implements Stoppable {
94 private static final Log LOG = LogFactory.getLog(SnapshotManager.class);
95
96
97 private static final int SNAPSHOT_WAKE_MILLIS_DEFAULT = 500;
98
99
100
101
102
103
104
105
106
107
108
109
110 private static final int SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT = 60 * 1000;
111
112
113 public static final String HBASE_SNAPSHOT_ENABLED = "hbase.snapshot.enabled";
114
115
116
117
118
119 private static final String SNAPSHOT_WAKE_MILLIS_KEY = "hbase.snapshot.master.wakeMillis";
120
121
122 private static final int SNAPSHOT_TIMEOUT_MILLIS_DEFAULT = 60000;
123
124
125
126
127
128 private static final String SNAPSHOT_TIMEOUT_MILLIS_KEY = "hbase.snapshot.master.timeoutMillis";
129
130
131 public static final String ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION = "online-snapshot";
132
133
134 private static final String SNAPSHOT_POOL_THREADS_KEY = "hbase.snapshot.master.threads";
135
136
137 private static final int SNAPSHOT_POOL_THREADS_DEFAULT = 1;
138
139 private boolean stopped;
140 private MasterServices master;
141 private MetricsMaster metricsMaster;
142 private ProcedureCoordinator coordinator;
143
144
145 private boolean isSnapshotSupported = false;
146
147
148
149
150
151 private Map<TableName, SnapshotSentinel> snapshotHandlers =
152 new HashMap<TableName, SnapshotSentinel>();
153
154
155
156
157
158 private Map<TableName, SnapshotSentinel> restoreHandlers =
159 new HashMap<TableName, SnapshotSentinel>();
160
161 private Path rootDir;
162 private ExecutorService executorService;
163
164
165
166
167 private int snapshotLayoutVersion = SnapshotDescriptionUtils.SNAPSHOT_LAYOUT_LATEST_FORMAT;
168
169 public SnapshotManager() {}
170
171
172
173
174
175
176
177 public SnapshotManager(final MasterServices master, final MetricsMaster metricsMaster,
178 ProcedureCoordinator coordinator, ExecutorService pool)
179 throws IOException, UnsupportedOperationException {
180 this.master = master;
181 this.metricsMaster = metricsMaster;
182
183 this.rootDir = master.getMasterFileSystem().getRootDir();
184 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
185
186 this.snapshotLayoutVersion = SnapshotDescriptionUtils.getDefaultSnapshotLayoutFormat(
187 master.getConfiguration());
188
189 this.coordinator = coordinator;
190 this.executorService = pool;
191 resetTempDir();
192 }
193
194
195
196
197
198
199 public List<SnapshotDescription> getCompletedSnapshots() throws IOException {
200 return getCompletedSnapshots(SnapshotDescriptionUtils.getSnapshotsDir(rootDir));
201 }
202
203
204
205
206
207
208
209 private List<SnapshotDescription> getCompletedSnapshots(Path snapshotDir) throws IOException {
210 List<SnapshotDescription> snapshotDescs = new ArrayList<SnapshotDescription>();
211
212 FileSystem fs = master.getMasterFileSystem().getFileSystem();
213 if (snapshotDir == null) snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
214
215
216 if (!fs.exists(snapshotDir)) {
217 return snapshotDescs;
218 }
219
220
221 FileStatus[] snapshots = fs.listStatus(snapshotDir,
222 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
223
224 for (FileStatus snapshot : snapshots) {
225 Path info = new Path(snapshot.getPath(), SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
226
227 if (!fs.exists(info)) {
228 LOG.error("Snapshot information for " + snapshot.getPath() + " doesn't exist");
229 continue;
230 }
231 FSDataInputStream in = null;
232 try {
233 in = fs.open(info);
234 SnapshotDescription desc = SnapshotDescription.parseFrom(in);
235 snapshotDescs.add(desc);
236 } catch (IOException e) {
237 LOG.warn("Found a corrupted snapshot " + snapshot.getPath(), e);
238 } finally {
239 if (in != null) {
240 in.close();
241 }
242 }
243 }
244 return snapshotDescs;
245 }
246
247
248
249
250
251
252
253 void resetTempDir() throws IOException {
254
255 Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir);
256 if (master.getMasterFileSystem().getFileSystem().exists(tmpdir)) {
257 if (!master.getMasterFileSystem().getFileSystem().delete(tmpdir, true)) {
258 LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
259 }
260 }
261 }
262
263
264
265
266
267
268
269 public void deleteSnapshot(SnapshotDescription snapshot) throws SnapshotDoesNotExistException, IOException {
270
271
272 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
273 if (cpHost != null) {
274 cpHost.preDeleteSnapshot(snapshot);
275 }
276
277
278 if (!isSnapshotCompleted(snapshot)) {
279 throw new SnapshotDoesNotExistException(snapshot);
280 }
281
282 String snapshotName = snapshot.getName();
283 LOG.debug("Deleting snapshot: " + snapshotName);
284
285 MasterFileSystem fs = master.getMasterFileSystem();
286 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
287
288
289 if (!fs.getFileSystem().delete(snapshotDir, true)) {
290 throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
291 }
292
293
294 if (cpHost != null) {
295 cpHost.postDeleteSnapshot(snapshot);
296 }
297
298 }
299
300
301
302
303
304
305
306
307
308 public boolean isSnapshotDone(SnapshotDescription expected) throws IOException {
309
310 if (expected == null) {
311 throw new UnknownSnapshotException(
312 "No snapshot name passed in request, can't figure out which snapshot you want to check.");
313 }
314
315 String ssString = ClientSnapshotDescriptionUtils.toString(expected);
316
317
318
319 SnapshotSentinel handler = removeSentinelIfFinished(this.snapshotHandlers, expected);
320
321
322 cleanupSentinels();
323
324 if (handler == null) {
325
326
327
328
329
330
331 if (!isSnapshotCompleted(expected)) {
332 throw new UnknownSnapshotException("Snapshot " + ssString
333 + " is not currently running or one of the known completed snapshots.");
334 }
335
336 return true;
337 }
338
339
340 try {
341 handler.rethrowExceptionIfFailed();
342 } catch (ForeignException e) {
343
344 String status;
345 Procedure p = coordinator.getProcedure(expected.getName());
346 if (p != null) {
347 status = p.getStatus();
348 } else {
349 status = expected.getName() + " not found in proclist " + coordinator.getProcedureNames();
350 }
351 throw new HBaseSnapshotException("Snapshot " + ssString + " had an error. " + status, e,
352 expected);
353 }
354
355
356 if (handler.isFinished()) {
357 LOG.debug("Snapshot '" + ssString + "' has completed, notifying client.");
358 return true;
359 } else if (LOG.isDebugEnabled()) {
360 LOG.debug("Snapshoting '" + ssString + "' is still in progress!");
361 }
362 return false;
363 }
364
365
366
367
368
369
370
371
372
373 synchronized boolean isTakingSnapshot(final SnapshotDescription snapshot) {
374 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
375 if (isTakingSnapshot(snapshotTable)) {
376 return true;
377 }
378 Iterator<Map.Entry<TableName, SnapshotSentinel>> it = this.snapshotHandlers.entrySet().iterator();
379 while (it.hasNext()) {
380 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
381 SnapshotSentinel sentinel = entry.getValue();
382 if (snapshot.getName().equals(sentinel.getSnapshot().getName()) && !sentinel.isFinished()) {
383 return true;
384 }
385 }
386 return false;
387 }
388
389
390
391
392
393
394
395 synchronized boolean isTakingSnapshot(final TableName tableName) {
396 SnapshotSentinel handler = this.snapshotHandlers.get(tableName);
397 return handler != null && !handler.isFinished();
398 }
399
400
401
402
403
404
405
406 private synchronized void prepareToTakeSnapshot(SnapshotDescription snapshot)
407 throws HBaseSnapshotException {
408 FileSystem fs = master.getMasterFileSystem().getFileSystem();
409 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
410 TableName snapshotTable =
411 TableName.valueOf(snapshot.getTable());
412
413
414 if (isTakingSnapshot(snapshot)) {
415 SnapshotSentinel handler = this.snapshotHandlers.get(snapshotTable);
416 throw new SnapshotCreationException("Rejected taking "
417 + ClientSnapshotDescriptionUtils.toString(snapshot)
418 + " because we are already running another snapshot "
419 + (handler != null ? ("on the same table " +
420 ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()))
421 : "with the same name"), snapshot);
422 }
423
424
425 if (isRestoringTable(snapshotTable)) {
426 SnapshotSentinel handler = restoreHandlers.get(snapshotTable);
427 throw new SnapshotCreationException("Rejected taking "
428 + ClientSnapshotDescriptionUtils.toString(snapshot)
429 + " because we are already have a restore in progress on the same snapshot "
430 + ClientSnapshotDescriptionUtils.toString(handler.getSnapshot()), snapshot);
431 }
432
433 try {
434
435
436 fs.delete(workingDir, true);
437
438
439 if (!fs.mkdirs(workingDir)) {
440 throw new SnapshotCreationException("Couldn't create working directory (" + workingDir
441 + ") for snapshot" , snapshot);
442 }
443 } catch (HBaseSnapshotException e) {
444 throw e;
445 } catch (IOException e) {
446 throw new SnapshotCreationException(
447 "Exception while checking to see if snapshot could be started.", e, snapshot);
448 }
449 }
450
451
452
453
454
455
456 private synchronized void snapshotDisabledTable(SnapshotDescription snapshot)
457 throws HBaseSnapshotException {
458
459 prepareToTakeSnapshot(snapshot);
460
461
462 snapshot = snapshot.toBuilder().setType(Type.DISABLED).build();
463
464
465 DisabledTableSnapshotHandler handler =
466 new DisabledTableSnapshotHandler(snapshot, master);
467 snapshotTable(snapshot, handler);
468 }
469
470
471
472
473
474
475 private synchronized void snapshotEnabledTable(SnapshotDescription snapshot)
476 throws HBaseSnapshotException {
477
478 prepareToTakeSnapshot(snapshot);
479
480
481 EnabledTableSnapshotHandler handler =
482 new EnabledTableSnapshotHandler(snapshot, master, this);
483 snapshotTable(snapshot, handler);
484 }
485
486
487
488
489
490
491
492
493
494 private synchronized void snapshotTable(SnapshotDescription snapshot,
495 final TakeSnapshotHandler handler) throws HBaseSnapshotException {
496 try {
497 handler.prepare();
498 this.executorService.submit(handler);
499 this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
500 } catch (Exception e) {
501
502 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir);
503 try {
504 if (!this.master.getMasterFileSystem().getFileSystem().delete(workingDir, true)) {
505 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
506 ClientSnapshotDescriptionUtils.toString(snapshot));
507 }
508 } catch (IOException e1) {
509 LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
510 ClientSnapshotDescriptionUtils.toString(snapshot));
511 }
512
513 throw new SnapshotCreationException("Could not build snapshot handler", e, snapshot);
514 }
515 }
516
517
518
519
520
521
522
523
524 public void takeSnapshot(SnapshotDescription snapshot) throws IOException {
525
526 if (isSnapshotCompleted(snapshot)) {
527 throw new SnapshotExistsException("Snapshot '" + snapshot.getName()
528 + "' already stored on the filesystem.", snapshot);
529 }
530
531 LOG.debug("No existing snapshot, attempting snapshot...");
532
533
534 cleanupSentinels();
535
536
537 HTableDescriptor desc = null;
538 try {
539 desc = master.getTableDescriptors().get(
540 TableName.valueOf(snapshot.getTable()));
541 } catch (FileNotFoundException e) {
542 String msg = "Table:" + snapshot.getTable() + " info doesn't exist!";
543 LOG.error(msg);
544 throw new SnapshotCreationException(msg, e, snapshot);
545 } catch (IOException e) {
546 throw new SnapshotCreationException("Error while geting table description for table "
547 + snapshot.getTable(), e, snapshot);
548 }
549 if (desc == null) {
550 throw new SnapshotCreationException("Table '" + snapshot.getTable()
551 + "' doesn't exist, can't take snapshot.", snapshot);
552 }
553
554
555 if (!snapshot.hasVersion()) {
556 snapshot = snapshot.toBuilder()
557 .setVersion(snapshotLayoutVersion)
558 .build();
559 }
560
561
562 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
563 if (cpHost != null) {
564 cpHost.preSnapshot(snapshot, desc);
565 }
566
567
568 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
569 AssignmentManager assignmentMgr = master.getAssignmentManager();
570 if (assignmentMgr.getZKTable().isEnabledTable(snapshotTable)) {
571 LOG.debug("Table enabled, starting distributed snapshot.");
572 snapshotEnabledTable(snapshot);
573 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
574 }
575
576 else if (assignmentMgr.getZKTable().isDisabledTable(snapshotTable)) {
577 LOG.debug("Table is disabled, running snapshot entirely on master.");
578 snapshotDisabledTable(snapshot);
579 LOG.debug("Started snapshot: " + ClientSnapshotDescriptionUtils.toString(snapshot));
580 } else {
581 LOG.error("Can't snapshot table '" + snapshot.getTable()
582 + "', isn't open or closed, we don't know what to do!");
583 TablePartiallyOpenException tpoe = new TablePartiallyOpenException(snapshot.getTable()
584 + " isn't fully open.");
585 throw new SnapshotCreationException("Table is not entirely open or closed", tpoe, snapshot);
586 }
587
588
589 if (cpHost != null) {
590 cpHost.postSnapshot(snapshot, desc);
591 }
592 }
593
594
595
596
597
598
599
600
601
602
603 public synchronized void setSnapshotHandlerForTesting(
604 final TableName tableName,
605 final SnapshotSentinel handler) {
606 if (handler != null) {
607 this.snapshotHandlers.put(tableName, handler);
608 } else {
609 this.snapshotHandlers.remove(tableName);
610 }
611 }
612
613
614
615
616 ProcedureCoordinator getCoordinator() {
617 return coordinator;
618 }
619
620
621
622
623
624
625
626
627
628
629
630 private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
631 try {
632 final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
633 FileSystem fs = master.getMasterFileSystem().getFileSystem();
634
635 return fs.exists(snapshotDir);
636 } catch (IllegalArgumentException iae) {
637 throw new UnknownSnapshotException("Unexpected exception thrown", iae);
638 }
639 }
640
641
642
643
644
645
646
647
648 synchronized void cloneSnapshot(final SnapshotDescription snapshot,
649 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
650 TableName tableName = hTableDescriptor.getTableName();
651
652
653 if (isTakingSnapshot(tableName)) {
654 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
655 }
656
657
658 if (isRestoringTable(tableName)) {
659 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
660 }
661
662 try {
663 CloneSnapshotHandler handler =
664 new CloneSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
665 this.executorService.submit(handler);
666 this.restoreHandlers.put(tableName, handler);
667 } catch (Exception e) {
668 String msg = "Couldn't clone the snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
669 " on table=" + tableName;
670 LOG.error(msg, e);
671 throw new RestoreSnapshotException(msg, e);
672 }
673 }
674
675
676
677
678
679
680 public void restoreSnapshot(SnapshotDescription reqSnapshot) throws IOException {
681 FileSystem fs = master.getMasterFileSystem().getFileSystem();
682 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
683 MasterCoprocessorHost cpHost = master.getCoprocessorHost();
684
685
686 if (!fs.exists(snapshotDir)) {
687 LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
688 throw new SnapshotDoesNotExistException(reqSnapshot);
689 }
690
691
692 SnapshotDescription fsSnapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
693 SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
694 snapshotDir, fsSnapshot);
695 HTableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
696 TableName tableName = TableName.valueOf(reqSnapshot.getTable());
697
698
699 cleanupSentinels();
700
701
702 SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);
703
704
705 if (MetaReader.tableExists(master.getCatalogTracker(), tableName)) {
706 if (master.getAssignmentManager().getZKTable().isEnabledTable(
707 TableName.valueOf(fsSnapshot.getTable()))) {
708 throw new UnsupportedOperationException("Table '" +
709 TableName.valueOf(fsSnapshot.getTable()) + "' must be disabled in order to " +
710 "perform a restore operation" +
711 ".");
712 }
713
714
715 if (cpHost != null) {
716 cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc);
717 }
718 restoreSnapshot(fsSnapshot, snapshotTableDesc);
719 LOG.info("Restore snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
720
721 if (cpHost != null) {
722 cpHost.postRestoreSnapshot(reqSnapshot, snapshotTableDesc);
723 }
724 } else {
725 HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName);
726 if (cpHost != null) {
727 cpHost.preCloneSnapshot(reqSnapshot, htd);
728 }
729 cloneSnapshot(fsSnapshot, htd);
730 LOG.info("Clone snapshot=" + fsSnapshot.getName() + " as table=" + tableName);
731
732 if (cpHost != null) {
733 cpHost.postCloneSnapshot(reqSnapshot, htd);
734 }
735 }
736 }
737
738
739
740
741
742
743
744
745 private synchronized void restoreSnapshot(final SnapshotDescription snapshot,
746 final HTableDescriptor hTableDescriptor) throws HBaseSnapshotException {
747 TableName tableName = hTableDescriptor.getTableName();
748
749
750 if (isTakingSnapshot(tableName)) {
751 throw new RestoreSnapshotException("Snapshot in progress on the restore table=" + tableName);
752 }
753
754
755 if (isRestoringTable(tableName)) {
756 throw new RestoreSnapshotException("Restore already in progress on the table=" + tableName);
757 }
758
759 try {
760 RestoreSnapshotHandler handler =
761 new RestoreSnapshotHandler(master, snapshot, hTableDescriptor).prepare();
762 this.executorService.submit(handler);
763 restoreHandlers.put(tableName, handler);
764 } catch (Exception e) {
765 String msg = "Couldn't restore the snapshot=" + ClientSnapshotDescriptionUtils.toString(
766 snapshot) +
767 " on table=" + tableName;
768 LOG.error(msg, e);
769 throw new RestoreSnapshotException(msg, e);
770 }
771 }
772
773
774
775
776
777
778
779 private synchronized boolean isRestoringTable(final TableName tableName) {
780 SnapshotSentinel sentinel = this.restoreHandlers.get(tableName);
781 return(sentinel != null && !sentinel.isFinished());
782 }
783
784
785
786
787
788
789
790
791
792 public boolean isRestoreDone(final SnapshotDescription snapshot) throws IOException {
793
794
795 SnapshotSentinel sentinel = removeSentinelIfFinished(this.restoreHandlers, snapshot);
796
797
798 cleanupSentinels();
799
800 if (sentinel == null) {
801
802 return true;
803 }
804
805 LOG.debug("Verify snapshot=" + snapshot.getName() + " against="
806 + sentinel.getSnapshot().getName() + " table=" +
807 TableName.valueOf(snapshot.getTable()));
808
809
810 sentinel.rethrowExceptionIfFailed();
811
812
813 if (sentinel.isFinished()) {
814 LOG.debug("Restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot) +
815 " has completed. Notifying the client.");
816 return true;
817 }
818
819 if (LOG.isDebugEnabled()) {
820 LOG.debug("Sentinel is not yet finished with restoring snapshot=" +
821 ClientSnapshotDescriptionUtils.toString(snapshot));
822 }
823 return false;
824 }
825
826
827
828
829
830
831
832
833 private synchronized SnapshotSentinel removeSentinelIfFinished(
834 final Map<TableName, SnapshotSentinel> sentinels,
835 final SnapshotDescription snapshot) {
836 if (!snapshot.hasTable()) {
837 return null;
838 }
839
840 TableName snapshotTable = TableName.valueOf(snapshot.getTable());
841 SnapshotSentinel h = sentinels.get(snapshotTable);
842 if (h == null) {
843 return null;
844 }
845
846 if (!h.getSnapshot().getName().equals(snapshot.getName())) {
847
848 return null;
849 }
850
851
852 if (h.isFinished()) {
853 sentinels.remove(snapshotTable);
854 }
855
856 return h;
857 }
858
859
860
861
862
863
864
865
866 private void cleanupSentinels() {
867 cleanupSentinels(this.snapshotHandlers);
868 cleanupSentinels(this.restoreHandlers);
869 }
870
871
872
873
874
875
876 private synchronized void cleanupSentinels(final Map<TableName, SnapshotSentinel> sentinels) {
877 long currentTime = EnvironmentEdgeManager.currentTimeMillis();
878 Iterator<Map.Entry<TableName, SnapshotSentinel>> it =
879 sentinels.entrySet().iterator();
880 while (it.hasNext()) {
881 Map.Entry<TableName, SnapshotSentinel> entry = it.next();
882 SnapshotSentinel sentinel = entry.getValue();
883 if (sentinel.isFinished() &&
884 (currentTime - sentinel.getCompletionTimestamp()) > SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT)
885 {
886 it.remove();
887 }
888 }
889 }
890
891
892
893
894
895 @Override
896 public void stop(String why) {
897
898 if (this.stopped) return;
899
900 this.stopped = true;
901
902 for (SnapshotSentinel snapshotHandler: this.snapshotHandlers.values()) {
903 snapshotHandler.cancel(why);
904 }
905
906
907 for (SnapshotSentinel restoreHandler: this.restoreHandlers.values()) {
908 restoreHandler.cancel(why);
909 }
910 try {
911 if (coordinator != null) {
912 coordinator.close();
913 }
914 } catch (IOException e) {
915 LOG.error("stop ProcedureCoordinator error", e);
916 }
917 }
918
919 @Override
920 public boolean isStopped() {
921 return this.stopped;
922 }
923
924
925
926
927
928
929 public void checkSnapshotSupport() throws UnsupportedOperationException {
930 if (!this.isSnapshotSupported) {
931 throw new UnsupportedOperationException(
932 "To use snapshots, You must add to the hbase-site.xml of the HBase Master: '" +
933 HBASE_SNAPSHOT_ENABLED + "' property with value 'true'.");
934 }
935 }
936
937
938
939
940
941
942
943
944
945
946
947 private void checkSnapshotSupport(final Configuration conf, final MasterFileSystem mfs)
948 throws IOException, UnsupportedOperationException {
949
950 String enabled = conf.get(HBASE_SNAPSHOT_ENABLED);
951 boolean snapshotEnabled = conf.getBoolean(HBASE_SNAPSHOT_ENABLED, false);
952 boolean userDisabled = (enabled != null && enabled.trim().length() > 0 && !snapshotEnabled);
953
954
955 Set<String> hfileCleaners = new HashSet<String>();
956 String[] cleaners = conf.getStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS);
957 if (cleaners != null) Collections.addAll(hfileCleaners, cleaners);
958
959 Set<String> logCleaners = new HashSet<String>();
960 cleaners = conf.getStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
961 if (cleaners != null) Collections.addAll(logCleaners, cleaners);
962
963
964 Path oldSnapshotDir = new Path(mfs.getRootDir(), HConstants.OLD_SNAPSHOT_DIR_NAME);
965 FileSystem fs = mfs.getFileSystem();
966 List<SnapshotDescription> ss = getCompletedSnapshots(new Path(rootDir, oldSnapshotDir));
967 if (ss != null && !ss.isEmpty()) {
968 LOG.error("Snapshots from an earlier release were found under: " + oldSnapshotDir);
969 LOG.error("Please rename the directory as " + HConstants.SNAPSHOT_DIR_NAME);
970 }
971
972
973
974
975 if (snapshotEnabled) {
976
977 hfileCleaners.add(SnapshotHFileCleaner.class.getName());
978 hfileCleaners.add(HFileLinkCleaner.class.getName());
979 logCleaners.add(SnapshotLogCleaner.class.getName());
980
981
982 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
983 hfileCleaners.toArray(new String[hfileCleaners.size()]));
984 conf.setStrings(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS,
985 logCleaners.toArray(new String[logCleaners.size()]));
986 } else {
987
988 snapshotEnabled = logCleaners.contains(SnapshotLogCleaner.class.getName()) &&
989 hfileCleaners.contains(SnapshotHFileCleaner.class.getName()) &&
990 hfileCleaners.contains(HFileLinkCleaner.class.getName());
991
992
993 if (snapshotEnabled) {
994 LOG.warn("Snapshot log and hfile cleaners are present in the configuration, " +
995 "but the '" + HBASE_SNAPSHOT_ENABLED + "' property " +
996 (userDisabled ? "is set to 'false'." : "is not set."));
997 }
998 }
999
1000
1001 this.isSnapshotSupported = snapshotEnabled && !userDisabled;
1002
1003
1004
1005 if (!snapshotEnabled) {
1006 LOG.info("Snapshot feature is not enabled, missing log and hfile cleaners.");
1007 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(mfs.getRootDir());
1008 if (fs.exists(snapshotDir)) {
1009 FileStatus[] snapshots = FSUtils.listStatus(fs, snapshotDir,
1010 new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
1011 if (snapshots != null) {
1012 LOG.error("Snapshots are present, but cleaners are not enabled.");
1013 checkSnapshotSupport();
1014 }
1015 }
1016 }
1017 }
1018
1019 @Override
1020 public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
1021 IOException, UnsupportedOperationException {
1022 this.master = master;
1023 this.metricsMaster = metricsMaster;
1024
1025 this.rootDir = master.getMasterFileSystem().getRootDir();
1026 checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());
1027
1028
1029 Configuration conf = master.getConfiguration();
1030 long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
1031 long timeoutMillis = conf.getLong(SNAPSHOT_TIMEOUT_MILLIS_KEY, SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
1032 int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);
1033
1034
1035 String name = master.getServerName().toString();
1036 ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
1037 ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinatorRpcs(
1038 master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);
1039
1040 this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
1041 this.executorService = master.getExecutorService();
1042 resetTempDir();
1043 }
1044
1045 @Override
1046 public String getProcedureSignature() {
1047 return ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION;
1048 }
1049
1050 @Override
1051 public void execProcedure(ProcedureDescription desc) throws IOException {
1052 takeSnapshot(toSnapshotDescription(desc));
1053 }
1054
1055 @Override
1056 public boolean isProcedureDone(ProcedureDescription desc) throws IOException {
1057 return isSnapshotDone(toSnapshotDescription(desc));
1058 }
1059
1060 private SnapshotDescription toSnapshotDescription(ProcedureDescription desc)
1061 throws IOException {
1062 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
1063 if (!desc.hasInstance()) {
1064 throw new IOException("Snapshot name is not defined: " + desc.toString());
1065 }
1066 String snapshotName = desc.getInstance();
1067 List<NameStringPair> props = desc.getConfigurationList();
1068 String table = null;
1069 for (NameStringPair prop : props) {
1070 if ("table".equalsIgnoreCase(prop.getName())) {
1071 table = prop.getValue();
1072 }
1073 }
1074 if (table == null) {
1075 throw new IOException("Snapshot table is not defined: " + desc.toString());
1076 }
1077 TableName tableName = TableName.valueOf(table);
1078 builder.setTable(tableName.getNameAsString());
1079 builder.setName(snapshotName);
1080 builder.setType(SnapshotDescription.Type.FLUSH);
1081 return builder.build();
1082 }
1083 }