1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.NavigableSet;
25
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FSDataOutputStream;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
32 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.EntryBuffers;
33 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.RegionEntryBuffer;
34 import org.apache.hadoop.hbase.testclassification.SmallTests;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38
39 import static org.mockito.Mockito.mock;
40
41
42
43
44 @Category(SmallTests.class)
45 public class TestHLogMethods {
46 private static final byte[] TEST_REGION = Bytes.toBytes("test_region");;
47 private static final TableName TEST_TABLE =
48 TableName.valueOf("test_table");
49
50 private final HBaseTestingUtility util = new HBaseTestingUtility();
51
52
53
54
55
56
57 @Test public void testGetSplitEditFilesSorted() throws IOException {
58 FileSystem fs = FileSystem.get(util.getConfiguration());
59 Path regiondir = util.getDataTestDir("regiondir");
60 fs.delete(regiondir, true);
61 fs.mkdirs(regiondir);
62 Path recoverededits = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
63 String first = HLogSplitter.formatRecoveredEditsFileName(-1);
64 createFile(fs, recoverededits, first);
65 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(0));
66 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(1));
67 createFile(fs, recoverededits, HLogSplitter
68 .formatRecoveredEditsFileName(11));
69 createFile(fs, recoverededits, HLogSplitter.formatRecoveredEditsFileName(2));
70 createFile(fs, recoverededits, HLogSplitter
71 .formatRecoveredEditsFileName(50));
72 String last = HLogSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE);
73 createFile(fs, recoverededits, last);
74 createFile(fs, recoverededits,
75 Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis());
76
77 HLogFactory.createHLog(fs, regiondir, "dummyLogName", util.getConfiguration());
78 NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regiondir);
79 assertEquals(7, files.size());
80 assertEquals(files.pollFirst().getName(), first);
81 assertEquals(files.pollLast().getName(), last);
82 assertEquals(files.pollFirst().getName(),
83 HLogSplitter
84 .formatRecoveredEditsFileName(0));
85 assertEquals(files.pollFirst().getName(),
86 HLogSplitter
87 .formatRecoveredEditsFileName(1));
88 assertEquals(files.pollFirst().getName(),
89 HLogSplitter
90 .formatRecoveredEditsFileName(2));
91 assertEquals(files.pollFirst().getName(),
92 HLogSplitter
93 .formatRecoveredEditsFileName(11));
94 }
95
96 private void createFile(final FileSystem fs, final Path testdir,
97 final String name)
98 throws IOException {
99 FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
100 fdos.close();
101 }
102
103 @Test
104 public void testRegionEntryBuffer() throws Exception {
105 HLogSplitter.RegionEntryBuffer reb = new HLogSplitter.RegionEntryBuffer(
106 TEST_TABLE, TEST_REGION);
107 assertEquals(0, reb.heapSize());
108
109 reb.appendEntry(createTestLogEntry(1));
110 assertTrue(reb.heapSize() > 0);
111 }
112
113 @Test
114 public void testEntrySink() throws Exception {
115 Configuration conf = new Configuration();
116 RecoveryMode mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
117 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
118 HLogSplitter splitter = new HLogSplitter(
119 conf, mock(Path.class), mock(FileSystem.class), null, null, mode);
120
121 EntryBuffers sink = splitter.new EntryBuffers(1*1024*1024);
122 for (int i = 0; i < 1000; i++) {
123 HLog.Entry entry = createTestLogEntry(i);
124 sink.appendEntry(entry);
125 }
126
127 assertTrue(sink.totalBuffered > 0);
128 long amountInChunk = sink.totalBuffered;
129
130 RegionEntryBuffer chunk = sink.getChunkToWrite();
131 assertEquals(chunk.heapSize(), amountInChunk);
132
133
134 assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
135
136
137 for (int i = 0; i < 500; i++) {
138 HLog.Entry entry = createTestLogEntry(i);
139 sink.appendEntry(entry);
140 }
141
142
143 assertNull(sink.getChunkToWrite());
144
145
146
147 sink.doneWriting(chunk);
148
149 RegionEntryBuffer chunk2 = sink.getChunkToWrite();
150 assertNotNull(chunk2);
151 assertNotSame(chunk, chunk2);
152 long amountInChunk2 = sink.totalBuffered;
153
154 assertTrue(amountInChunk2 < amountInChunk);
155
156 sink.doneWriting(chunk2);
157 assertEquals(0, sink.totalBuffered);
158 }
159
160 private HLog.Entry createTestLogEntry(int i) {
161 long seq = i;
162 long now = i * 1000;
163
164 WALEdit edit = new WALEdit();
165 edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
166 HLogKey key = new HLogKey(TEST_REGION, TEST_TABLE, seq, now,
167 HConstants.DEFAULT_CLUSTER_ID);
168 HLog.Entry entry = new HLog.Entry(key, edit);
169 return entry;
170 }
171
172 }
173