1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.util.Arrays;
30 import java.util.Comparator;
31 import java.util.Map;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.conf.Configuration;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.TableDescriptors;
46 import org.apache.hadoop.hbase.TableExistsException;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49
50
51
52
53
54
55 @Category(MediumTests.class)
56 public class TestFSTableDescriptors {
57 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
58 private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
59
60 @Test (expected=IllegalArgumentException.class)
61 public void testRegexAgainstOldStyleTableInfo() {
62 Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX);
63 int i = FSTableDescriptors.getTableInfoSequenceId(p);
64 assertEquals(0, i);
65
66 p = new Path("/tmp", "abc");
67 FSTableDescriptors.getTableInfoSequenceId(p);
68 }
69
70 @Test
71 public void testCreateAndUpdate() throws IOException {
72 Path testdir = UTIL.getDataTestDir("testCreateAndUpdate");
73 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testCreate"));
74 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
75 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
76 assertTrue(fstd.createTableDescriptor(htd));
77 assertFalse(fstd.createTableDescriptor(htd));
78 FileStatus [] statuses = fs.listStatus(testdir);
79 assertTrue("statuses.length="+statuses.length, statuses.length == 1);
80 for (int i = 0; i < 10; i++) {
81 fstd.updateTableDescriptor(htd);
82 }
83 statuses = fs.listStatus(testdir);
84 assertTrue(statuses.length == 1);
85 Path tmpTableDir = new Path(FSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
86 statuses = fs.listStatus(tmpTableDir);
87 assertTrue(statuses.length == 0);
88 }
89
90 @Test
91 public void testSequenceIdAdvancesOnTableInfo() throws IOException {
92 Path testdir = UTIL.getDataTestDir("testSequenceidAdvancesOnTableInfo");
93 HTableDescriptor htd = new HTableDescriptor(
94 TableName.valueOf("testSequenceidAdvancesOnTableInfo"));
95 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
96 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
97 Path p0 = fstd.updateTableDescriptor(htd);
98 int i0 = FSTableDescriptors.getTableInfoSequenceId(p0);
99 Path p1 = fstd.updateTableDescriptor(htd);
100
101 assertTrue(!fs.exists(p0));
102 int i1 = FSTableDescriptors.getTableInfoSequenceId(p1);
103 assertTrue(i1 == i0 + 1);
104 Path p2 = fstd.updateTableDescriptor(htd);
105
106 assertTrue(!fs.exists(p1));
107 int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
108 assertTrue(i2 == i1 + 1);
109 }
110
111 @Test
112 public void testFormatTableInfoSequenceId() {
113 Path p0 = assertWriteAndReadSequenceId(0);
114
115 StringBuilder sb = new StringBuilder();
116 for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
117 sb.append("0");
118 }
119 assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
120 p0.getName());
121
122 Path p2 = assertWriteAndReadSequenceId(2);
123 Path p10000 = assertWriteAndReadSequenceId(10000);
124
125 Path p = new Path(p0.getParent(), FSTableDescriptors.TABLEINFO_FILE_PREFIX);
126 FileStatus fs = new FileStatus(0, false, 0, 0, 0, p);
127 FileStatus fs0 = new FileStatus(0, false, 0, 0, 0, p0);
128 FileStatus fs2 = new FileStatus(0, false, 0, 0, 0, p2);
129 FileStatus fs10000 = new FileStatus(0, false, 0, 0, 0, p10000);
130 Comparator<FileStatus> comparator = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
131 assertTrue(comparator.compare(fs, fs0) > 0);
132 assertTrue(comparator.compare(fs0, fs2) > 0);
133 assertTrue(comparator.compare(fs2, fs10000) > 0);
134 }
135
136 private Path assertWriteAndReadSequenceId(final int i) {
137 Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i));
138 int ii = FSTableDescriptors.getTableInfoSequenceId(p);
139 assertEquals(i, ii);
140 return p;
141 }
142
143 @Test
144 public void testRemoves() throws IOException {
145 final String name = "testRemoves";
146 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
147
148 Path rootdir = new Path(UTIL.getDataTestDir(), name);
149 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
150 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
151 htds.add(htd);
152 assertNotNull(htds.remove(htd.getTableName()));
153 assertNull(htds.remove(htd.getTableName()));
154 }
155
156 @Test public void testReadingHTDFromFS() throws IOException {
157 final String name = "testReadingHTDFromFS";
158 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
159 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
160 Path rootdir = UTIL.getDataTestDir(name);
161 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
162 fstd.createTableDescriptor(htd);
163 HTableDescriptor htd2 =
164 FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName());
165 assertTrue(htd.equals(htd2));
166 }
167
168 @Test public void testHTableDescriptors()
169 throws IOException, InterruptedException {
170 final String name = "testHTableDescriptors";
171 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
172
173 Path rootdir = new Path(UTIL.getDataTestDir(), name);
174 FSTableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir) {
175 @Override
176 public HTableDescriptor get(TableName tablename)
177 throws TableExistsException, FileNotFoundException, IOException {
178 LOG.info(tablename + ", cachehits=" + this.cachehits);
179 return super.get(tablename);
180 }
181 };
182 final int count = 10;
183
184 for (int i = 0; i < count; i++) {
185 HTableDescriptor htd = new HTableDescriptor(name + i);
186 htds.createTableDescriptor(htd);
187 }
188
189 for (int i = 0; i < count; i++) {
190 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
191 }
192 for (int i = 0; i < count; i++) {
193 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
194 }
195
196 for (int i = 0; i < count; i++) {
197 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
198 htd.addFamily(new HColumnDescriptor("" + i));
199 htds.updateTableDescriptor(htd);
200 }
201
202 Thread.sleep(100);
203 for (int i = 0; i < count; i++) {
204 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
205 }
206 for (int i = 0; i < count; i++) {
207 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
208 }
209 assertEquals(count * 4, htds.invocations);
210 assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
211 htds.cachehits >= (count * 2));
212 }
213 @Test
214 public void testHTableDescriptorsNoCache()
215 throws IOException, InterruptedException {
216 final String name = "testHTableDescriptorsNoCache";
217 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
218
219 Path rootdir = new Path(UTIL.getDataTestDir(), name);
220 FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir,
221 false, false);
222 final int count = 10;
223
224 for (int i = 0; i < count; i++) {
225 HTableDescriptor htd = new HTableDescriptor(name + i);
226 htds.createTableDescriptor(htd);
227 }
228
229 for (int i = 0; i < count; i++) {
230 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
231 }
232 for (int i = 0; i < count; i++) {
233 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
234 }
235
236 for (int i = 0; i < count; i++) {
237 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
238 htd.addFamily(new HColumnDescriptor("" + i));
239 htds.updateTableDescriptor(htd);
240 }
241
242 Thread.sleep(100);
243 for (int i = 0; i < count; i++) {
244 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
245 }
246 for (int i = 0; i < count; i++) {
247 assertTrue(htds.get(TableName.valueOf(name + i)) != null);
248 }
249 assertEquals(count * 4, htds.invocations);
250 assertTrue("expected=0, actual=" + htds.cachehits,
251 htds.cachehits == 0);
252 }
253
254 @Test
255 public void testGetAll()
256 throws IOException, InterruptedException {
257 final String name = "testGetAll";
258 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
259
260 Path rootdir = new Path(UTIL.getDataTestDir(), name);
261 FSTableDescriptors htds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
262 final int count = 4;
263
264 for (int i = 0; i < count; i++) {
265 HTableDescriptor htd = new HTableDescriptor(name + i);
266 htds.createTableDescriptor(htd);
267 }
268
269 HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
270 htds.createTableDescriptor(htd);
271
272 assertTrue(htds.getAll().size() == count + 1);
273
274 }
275
276 @Test
277 public void testCacheConsistency()
278 throws IOException, InterruptedException {
279 final String name = "testCacheConsistency";
280 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
281
282 Path rootdir = new Path(UTIL.getDataTestDir(), name);
283 FSTableDescriptors chtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs, rootdir);
284 FSTableDescriptors nonchtds = new FSTableDescriptorsTest(UTIL.getConfiguration(), fs,
285 rootdir, false, false);
286
287 final int count = 10;
288
289 for (int i = 0; i < count; i++) {
290 HTableDescriptor htd = new HTableDescriptor(name + i);
291 nonchtds.createTableDescriptor(htd);
292 }
293
294
295 for (int i = 0; i < count; i++) {
296 assertTrue(chtds.get(TableName.valueOf(name + i)) != null);
297 }
298
299 assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
300
301
302 HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
303 nonchtds.createTableDescriptor(htd);
304
305
306 assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
307
308 for (Map.Entry entry: nonchtds.getAll().entrySet()) {
309 String t = (String) entry.getKey();
310 HTableDescriptor nchtd = (HTableDescriptor) entry.getValue();
311 assertTrue("expected " + htd.toString() +
312 " got: " + chtds.get(TableName.valueOf(t)).toString(),
313 (nchtd.equals(chtds.get(TableName.valueOf(t)))));
314 }
315 }
316
317 @Test
318 public void testNoSuchTable() throws IOException {
319 final String name = "testNoSuchTable";
320 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
321
322 Path rootdir = new Path(UTIL.getDataTestDir(), name);
323 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
324 assertNull("There shouldn't be any HTD for this table",
325 htds.get(TableName.valueOf("NoSuchTable")));
326 }
327
328 @Test
329 public void testUpdates() throws IOException {
330 final String name = "testUpdates";
331 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
332
333 Path rootdir = new Path(UTIL.getDataTestDir(), name);
334 TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir);
335 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
336 htds.add(htd);
337 htds.add(htd);
338 htds.add(htd);
339 }
340
341 @Test
342 public void testTableInfoFileStatusComparator() {
343 FileStatus bare =
344 new FileStatus(0, false, 0, 0, -1,
345 new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
346 FileStatus future =
347 new FileStatus(0, false, 0, 0, -1,
348 new Path("/tmp/tablinfo." + System.currentTimeMillis()));
349 FileStatus farFuture =
350 new FileStatus(0, false, 0, 0, -1,
351 new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
352 FileStatus [] alist = {bare, future, farFuture};
353 FileStatus [] blist = {bare, farFuture, future};
354 FileStatus [] clist = {farFuture, bare, future};
355 Comparator<FileStatus> c = FSTableDescriptors.TABLEINFO_FILESTATUS_COMPARATOR;
356 Arrays.sort(alist, c);
357 Arrays.sort(blist, c);
358 Arrays.sort(clist, c);
359
360 for (int i = 0; i < alist.length; i++) {
361 assertTrue(alist[i].equals(blist[i]));
362 assertTrue(blist[i].equals(clist[i]));
363 assertTrue(clist[i].equals(i == 0? farFuture: i == 1? future: bare));
364 }
365 }
366
367 @Test
368 public void testReadingInvalidDirectoryFromFS() throws IOException {
369 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
370 try {
371
372 new FSTableDescriptors(UTIL.getConfiguration(), fs,
373 FSUtils.getRootDir(UTIL.getConfiguration()))
374 .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
375 fail("Shouldn't be able to read a table descriptor for the archive directory.");
376 } catch (Exception e) {
377 LOG.debug("Correctly got error when reading a table descriptor from the archive directory: "
378 + e.getMessage());
379 }
380 }
381
382 @Test
383 public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
384 Path testdir = UTIL.getDataTestDir("testCreateTableDescriptorUpdatesIfThereExistsAlready");
385 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(
386 "testCreateTableDescriptorUpdatesIfThereExistsAlready"));
387 FileSystem fs = FileSystem.get(UTIL.getConfiguration());
388 FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir);
389 assertTrue(fstd.createTableDescriptor(htd));
390 assertFalse(fstd.createTableDescriptor(htd));
391 htd.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue"));
392 assertTrue(fstd.createTableDescriptor(htd));
393 Path tableDir = fstd.getTableDir(htd.getTableName());
394 Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
395 FileStatus[] statuses = fs.listStatus(tmpTableDir);
396 assertTrue(statuses.length == 0);
397
398 assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
399 }
400
401 private static class FSTableDescriptorsTest
402 extends FSTableDescriptors {
403
404 public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir)
405 throws IOException {
406 this(conf, fs, rootdir, false, true);
407 }
408
409 public FSTableDescriptorsTest(Configuration conf, FileSystem fs, Path rootdir,
410 boolean fsreadonly, boolean usecache)
411 throws IOException {
412 super(conf, fs, rootdir, fsreadonly, usecache);
413 }
414
415 @Override
416 public HTableDescriptor get(TableName tablename)
417 throws TableExistsException, FileNotFoundException, IOException {
418 LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
419 " HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
420 return super.get(tablename);
421 }
422 }
423 }
424