View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.util;
20  
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertFalse;
24  import static org.junit.Assert.assertNotSame;
25  import static org.junit.Assert.assertTrue;
26  
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.Map;
30  
31  import org.apache.commons.lang.ArrayUtils;
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.testclassification.MediumTests;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.util.RegionSplitter.HexStringSplit;
41  import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
42  import org.apache.hadoop.hbase.util.RegionSplitter.UniformSplit;
43  import org.junit.AfterClass;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Tests for {@link RegionSplitter}, which can create a pre-split table or do a
50   * rolling split of an existing table.
51   */
52  @Category(MediumTests.class)
53  public class TestRegionSplitter {
54      private final static Log LOG = LogFactory.getLog(TestRegionSplitter.class);
55      private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
56      private final static String CF_NAME = "SPLIT_TEST_CF";
57      private final static byte xFF = (byte) 0xff;
58  
59      @BeforeClass
60      public static void setup() throws Exception {
61          UTIL.startMiniCluster();
62      }
63  
64      @AfterClass
65      public static void teardown() throws Exception {
66          UTIL.shutdownMiniCluster();
67      }
68  
69      /**
70       * Test creating a pre-split table using the HexStringSplit algorithm.
71       */
72      @Test
73      public void testCreatePresplitTableHex() throws Exception {
74        final List<byte[]> expectedBounds = new ArrayList<byte[]>();
75        expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
76        expectedBounds.add("10000000".getBytes());
77        expectedBounds.add("20000000".getBytes());
78        expectedBounds.add("30000000".getBytes());
79        expectedBounds.add("40000000".getBytes());
80        expectedBounds.add("50000000".getBytes());
81        expectedBounds.add("60000000".getBytes());
82        expectedBounds.add("70000000".getBytes());
83        expectedBounds.add("80000000".getBytes());
84        expectedBounds.add("90000000".getBytes());
85        expectedBounds.add("a0000000".getBytes());
86        expectedBounds.add("b0000000".getBytes());
87        expectedBounds.add("c0000000".getBytes());
88        expectedBounds.add("d0000000".getBytes());
89        expectedBounds.add("e0000000".getBytes());
90        expectedBounds.add("f0000000".getBytes());
91            expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
92  
93            // Do table creation/pre-splitting and verification of region boundaries
94        preSplitTableAndVerify(expectedBounds,
95            HexStringSplit.class.getSimpleName(), "NewHexPresplitTable");
96      }
97  
98      /**
99       * Test creating a pre-split table using the UniformSplit algorithm.
100      */
101     @Test
102     public void testCreatePresplitTableUniform() throws Exception {
103       List<byte[]> expectedBounds = new ArrayList<byte[]>();
104       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
105       expectedBounds.add(new byte[] {      0x10, 0, 0, 0, 0, 0, 0, 0});
106       expectedBounds.add(new byte[] {      0x20, 0, 0, 0, 0, 0, 0, 0});
107       expectedBounds.add(new byte[] {      0x30, 0, 0, 0, 0, 0, 0, 0});
108       expectedBounds.add(new byte[] {      0x40, 0, 0, 0, 0, 0, 0, 0});
109       expectedBounds.add(new byte[] {      0x50, 0, 0, 0, 0, 0, 0, 0});
110       expectedBounds.add(new byte[] {      0x60, 0, 0, 0, 0, 0, 0, 0});
111       expectedBounds.add(new byte[] {      0x70, 0, 0, 0, 0, 0, 0, 0});
112       expectedBounds.add(new byte[] {(byte)0x80, 0, 0, 0, 0, 0, 0, 0});
113       expectedBounds.add(new byte[] {(byte)0x90, 0, 0, 0, 0, 0, 0, 0});
114       expectedBounds.add(new byte[] {(byte)0xa0, 0, 0, 0, 0, 0, 0, 0});
115       expectedBounds.add(new byte[] {(byte)0xb0, 0, 0, 0, 0, 0, 0, 0});
116       expectedBounds.add(new byte[] {(byte)0xc0, 0, 0, 0, 0, 0, 0, 0});
117       expectedBounds.add(new byte[] {(byte)0xd0, 0, 0, 0, 0, 0, 0, 0});
118       expectedBounds.add(new byte[] {(byte)0xe0, 0, 0, 0, 0, 0, 0, 0});
119       expectedBounds.add(new byte[] {(byte)0xf0, 0, 0, 0, 0, 0, 0, 0});
120       expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
121 
122       // Do table creation/pre-splitting and verification of region boundaries
123       preSplitTableAndVerify(expectedBounds, UniformSplit.class.getSimpleName(),
124         "NewUniformPresplitTable");
125     }
126 
127     /**
128      * Unit tests for the HexStringSplit algorithm. Makes sure it divides up the
129      * space of keys in the way that we expect.
130      */
131     @Test
132     public void unitTestHexStringSplit() {
133         HexStringSplit splitter = new HexStringSplit();
134         // Check splitting while starting from scratch
135 
136         byte[][] twoRegionsSplits = splitter.split(2);
137         assertEquals(1, twoRegionsSplits.length);
138     assertArrayEquals(twoRegionsSplits[0], "80000000".getBytes());
139 
140         byte[][] threeRegionsSplits = splitter.split(3);
141         assertEquals(2, threeRegionsSplits.length);
142         byte[] expectedSplit0 = "55555555".getBytes();
143         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
144         byte[] expectedSplit1 = "aaaaaaaa".getBytes();
145         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
146 
147         // Check splitting existing regions that have start and end points
148         byte[] splitPoint = splitter.split("10000000".getBytes(), "30000000".getBytes());
149         assertArrayEquals("20000000".getBytes(), splitPoint);
150 
151         byte[] lastRow = "ffffffff".getBytes();
152         assertArrayEquals(lastRow, splitter.lastRow());
153         byte[] firstRow = "00000000".getBytes();
154         assertArrayEquals(firstRow, splitter.firstRow());
155 
156         // Halfway between 00... and 20... should be 10...
157         splitPoint = splitter.split(firstRow, "20000000".getBytes());
158         assertArrayEquals(splitPoint, "10000000".getBytes());
159 
160         // Halfway between df... and ff... should be ef....
161         splitPoint = splitter.split("dfffffff".getBytes(), lastRow);
162         assertArrayEquals(splitPoint,"efffffff".getBytes());
163     }
164 
165     /**
166      * Unit tests for the UniformSplit algorithm. Makes sure it divides up the space of
167      * keys in the way that we expect.
168      */
169     @Test
170     public void unitTestUniformSplit() {
171         UniformSplit splitter = new UniformSplit();
172 
173         // Check splitting while starting from scratch
174         try {
175             splitter.split(1);
176             throw new AssertionError("Splitting into <2 regions should have thrown exception");
177         } catch (IllegalArgumentException e) { }
178 
179         byte[][] twoRegionsSplits = splitter.split(2);
180         assertEquals(1, twoRegionsSplits.length);
181         assertArrayEquals(twoRegionsSplits[0],
182             new byte[] { (byte) 0x80, 0, 0, 0, 0, 0, 0, 0 });
183 
184         byte[][] threeRegionsSplits = splitter.split(3);
185         assertEquals(2, threeRegionsSplits.length);
186         byte[] expectedSplit0 = new byte[] {0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55, 0x55};
187         assertArrayEquals(expectedSplit0, threeRegionsSplits[0]);
188         byte[] expectedSplit1 = new byte[] {(byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA,
189                 (byte)0xAA, (byte)0xAA, (byte)0xAA, (byte)0xAA};
190         assertArrayEquals(expectedSplit1, threeRegionsSplits[1]);
191 
192         // Check splitting existing regions that have start and end points
193         byte[] splitPoint = splitter.split(new byte[] {0x10}, new byte[] {0x30});
194         assertArrayEquals(new byte[] {0x20}, splitPoint);
195 
196         byte[] lastRow = new byte[] {xFF, xFF, xFF, xFF, xFF, xFF, xFF, xFF};
197         assertArrayEquals(lastRow, splitter.lastRow());
198         byte[] firstRow = ArrayUtils.EMPTY_BYTE_ARRAY;
199         assertArrayEquals(firstRow, splitter.firstRow());
200 
201         splitPoint = splitter.split(firstRow, new byte[] {0x20});
202         assertArrayEquals(splitPoint, new byte[] {0x10});
203 
204         splitPoint = splitter.split(new byte[] {(byte)0xdf, xFF, xFF, xFF, xFF,
205                 xFF, xFF, xFF}, lastRow);
206         assertArrayEquals(splitPoint,
207                 new byte[] {(byte)0xef, xFF, xFF, xFF, xFF, xFF, xFF, xFF});
208 
209         splitPoint = splitter.split(new byte[] {'a', 'a', 'a'}, new byte[] {'a', 'a', 'b'});
210         assertArrayEquals(splitPoint, new byte[] {'a', 'a', 'a', (byte)0x80 });
211     }
212 
213   @Test
214   public void testUserInput() {
215     SplitAlgorithm algo = new HexStringSplit();
216     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
217     assertFalse(splitFailsPrecondition(algo, "00", "AA")); // custom is fine
218     assertTrue(splitFailsPrecondition(algo, "AA", "00")); // range error
219     assertTrue(splitFailsPrecondition(algo, "AA", "AA")); // range error
220     assertFalse(splitFailsPrecondition(algo, "0", "2", 3)); // should be fine
221     assertFalse(splitFailsPrecondition(algo, "0", "A", 11)); // should be fine
222     assertTrue(splitFailsPrecondition(algo, "0", "A", 12)); // too granular
223 
224     algo = new UniformSplit();
225     assertFalse(splitFailsPrecondition(algo)); // default settings are fine
226     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\xAA")); // custom is fine
227     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\x00")); // range error
228     assertTrue(splitFailsPrecondition(algo, "\\xAA", "\\xAA")); // range error
229     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x02", 3)); // should be fine
230     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 11)); // should be fine
231     assertFalse(splitFailsPrecondition(algo, "\\x00", "\\x0A", 12)); // should be fine
232   }
233 
234   private boolean splitFailsPrecondition(SplitAlgorithm algo) {
235     return splitFailsPrecondition(algo, 100);
236   }
237 
238   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
239       String lastRow) {
240     return splitFailsPrecondition(algo, firstRow, lastRow, 100);
241   }
242 
243   private boolean splitFailsPrecondition(SplitAlgorithm algo, String firstRow,
244       String lastRow, int numRegions) {
245     algo.setFirstRow(firstRow);
246     algo.setLastRow(lastRow);
247     return splitFailsPrecondition(algo, numRegions);
248   }
249 
250   private boolean splitFailsPrecondition(SplitAlgorithm algo, int numRegions) {
251     try {
252       byte[][] s = algo.split(numRegions);
253       LOG.debug("split algo = " + algo);
254       if (s != null) {
255         StringBuilder sb = new StringBuilder();
256         for (byte[] b : s) {
257           sb.append(Bytes.toStringBinary(b) + "  ");
258         }
259         LOG.debug(sb.toString());
260       }
261       return false;
262     } catch (IllegalArgumentException e) {
263       return true;
264     } catch (IllegalStateException e) {
265       return true;
266     } catch (IndexOutOfBoundsException e) {
267       return true;
268     }
269   }
270 
271     /**
272      * Creates a pre-split table with expectedBounds.size()+1 regions, then
273      * verifies that the region boundaries are the same as the expected
274      * region boundaries in expectedBounds.
275      * @throws Various junit assertions
276      */
277     private void preSplitTableAndVerify(List<byte[]> expectedBounds,
278             String splitClass, String tableName) throws Exception {
279         final int numRegions = expectedBounds.size()-1;
280         final Configuration conf = UTIL.getConfiguration();
281         conf.setInt("split.count", numRegions);
282         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
283         RegionSplitter.createPresplitTable(tableName, splitAlgo,
284                 new String[] {CF_NAME}, conf);
285         verifyBounds(expectedBounds, tableName);
286     }
287 
288   @Test
289   public void noopRollingSplit() throws Exception {
290     final List<byte[]> expectedBounds = new ArrayList<byte[]>();
291     expectedBounds.add(ArrayUtils.EMPTY_BYTE_ARRAY);
292     rollingSplitAndVerify(TestRegionSplitter.class.getSimpleName(), "UniformSplit", expectedBounds);
293   }
294 
295     private void rollingSplitAndVerify(String tableName, String splitClass,
296             List<byte[]> expectedBounds)  throws Exception {
297         final Configuration conf = UTIL.getConfiguration();
298 
299         // Set this larger than the number of splits so RegionSplitter won't block
300         conf.setInt("split.outstanding", 5);
301         SplitAlgorithm splitAlgo = RegionSplitter.newSplitAlgoInstance(conf, splitClass);
302         RegionSplitter.rollingSplit(tableName, splitAlgo, conf);
303         verifyBounds(expectedBounds, tableName);
304     }
305 
306     private void verifyBounds(List<byte[]> expectedBounds, String tableName)
307             throws Exception {
308         // Get region boundaries from the cluster and verify their endpoints
309         final Configuration conf = UTIL.getConfiguration();
310         final int numRegions = expectedBounds.size()-1;
311         final HTable hTable = new HTable(conf, tableName.getBytes());
312         final Map<HRegionInfo, ServerName> regionInfoMap = hTable.getRegionLocations();
313         assertEquals(numRegions, regionInfoMap.size());
314         for (Map.Entry<HRegionInfo, ServerName> entry: regionInfoMap.entrySet()) {
315             final HRegionInfo regionInfo = entry.getKey();
316             byte[] regionStart = regionInfo.getStartKey();
317             byte[] regionEnd = regionInfo.getEndKey();
318 
319             // This region's start key should be one of the region boundaries
320             int startBoundaryIndex = indexOfBytes(expectedBounds, regionStart);
321             assertNotSame(-1, startBoundaryIndex);
322 
323             // This region's end key should be the region boundary that comes
324             // after the starting boundary.
325             byte[] expectedRegionEnd = expectedBounds.get(
326                     startBoundaryIndex+1);
327             assertEquals(0, Bytes.compareTo(regionEnd, expectedRegionEnd));
328         }
329     }
330 
331     /**
332      * List.indexOf() doesn't really work for a List<byte[]>, because byte[]
333      * doesn't override equals(). This method checks whether a list contains
334      * a given element by checking each element using the byte array
335      * comparator.
336      * @return the index of the first element that equals compareTo, or -1
337      * if no elements are equal.
338      */
339     static private int indexOfBytes(List<byte[]> list,  byte[] compareTo) {
340         int listIndex = 0;
341         for(byte[] elem: list) {
342             if(Bytes.BYTES_COMPARATOR.compare(elem, compareTo) == 0) {
343                 return listIndex;
344             }
345             listIndex++;
346         }
347         return -1;
348     }
349 
350 }
351