View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.net.InetSocketAddress;
28  import java.util.ArrayList;
29  import java.util.Arrays;
30  import java.util.Collection;
31  import java.util.HashMap;
32  import java.util.List;
33  import java.util.Map;
34  import java.util.Random;
35  import java.util.Set;
36  import java.util.concurrent.atomic.AtomicInteger;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.conf.Configuration;
41  import org.apache.hadoop.hbase.ClusterStatus;
42  import org.apache.hadoop.hbase.HBaseIOException;
43  import org.apache.hadoop.hbase.HBaseTestingUtility;
44  import org.apache.hadoop.hbase.HColumnDescriptor;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.HRegionInfo;
47  import org.apache.hadoop.hbase.HTableDescriptor;
48  import org.apache.hadoop.hbase.testclassification.MediumTests;
49  import org.apache.hadoop.hbase.MiniHBaseCluster;
50  import org.apache.hadoop.hbase.NamespaceDescriptor;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableName;
53  import org.apache.hadoop.hbase.client.HBaseAdmin;
54  import org.apache.hadoop.hbase.client.HTable;
55  import org.apache.hadoop.hbase.client.MetaScanner;
56  import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
59  import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
60  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
61  import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
62  import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
63  import org.apache.hadoop.hbase.regionserver.HRegion;
64  import org.apache.hadoop.hbase.regionserver.HRegionServer;
65  import org.apache.hadoop.hbase.util.Bytes;
66  import org.apache.hadoop.hbase.util.Pair;
67  import org.apache.zookeeper.KeeperException;
68  import org.junit.AfterClass;
69  import org.junit.BeforeClass;
70  import org.junit.Test;
71  import org.junit.experimental.categories.Category;
72  
73  
74  @Category(MediumTests.class)
75  public class TestRegionPlacement {
76    final static Log LOG = LogFactory.getLog(TestRegionPlacement.class);
77    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
78    private final static int SLAVES = 10;
79    private static HBaseAdmin admin;
80    private static RegionPlacementMaintainer rp;
81    private static Position[] positions = Position.values();
82    private int lastRegionOnPrimaryRSCount = 0;
83    private int REGION_NUM = 10;
84    private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
85        new HashMap<HRegionInfo, ServerName[]>();
86    private final static int PRIMARY = Position.PRIMARY.ordinal();
87    private final static int SECONDARY = Position.SECONDARY.ordinal();
88    private final static int TERTIARY = Position.TERTIARY.ordinal();
89  
90    @BeforeClass
91    public static void setupBeforeClass() throws Exception {
92      Configuration conf = TEST_UTIL.getConfiguration();
93      // Enable the favored nodes based load balancer
94      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
95          FavoredNodeLoadBalancer.class, LoadBalancer.class);
96      conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
97      TEST_UTIL.startMiniCluster(SLAVES);
98      admin = new HBaseAdmin(conf);
99      rp = new RegionPlacementMaintainer(conf);
100   }
101 
102   @AfterClass
103   public static void tearDownAfterClass() throws Exception {
104     TEST_UTIL.shutdownMiniCluster();
105   }
106 
107   @Test
108   public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException {
109     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
110     balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
111     List<ServerName> servers = new ArrayList<ServerName>();
112     for (int i = 0; i < SLAVES; i++) {
113       ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
114       servers.add(server);
115     }
116     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
117     HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
118     regions.add(region);
119     Map<ServerName,List<HRegionInfo>> assignmentMap = balancer.roundRobinAssignment(regions,
120         servers);
121     Set<ServerName> serverBefore = assignmentMap.keySet();
122     List<ServerName> favoredNodesBefore =
123         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
124     assertTrue(favoredNodesBefore.size() == 3);
125     // the primary RS should be the one that the balancer's assignment returns
126     assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(),
127         favoredNodesBefore.get(PRIMARY)));
128     // now remove the primary from the list of available servers
129     List<ServerName> removedServers = removeMatchingServers(serverBefore, servers);
130     // call roundRobinAssignment with the modified servers list
131     assignmentMap = balancer.roundRobinAssignment(regions, servers);
132     List<ServerName> favoredNodesAfter =
133         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
134     assertTrue(favoredNodesAfter.size() == 3);
135     // We don't expect the favored nodes assignments to change in multiple calls
136     // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign
137     // failures)
138     assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
139     Set<ServerName> serverAfter = assignmentMap.keySet();
140     // We expect the new RegionServer assignee to be one of the favored nodes
141     // chosen earlier.
142     assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
143                  favoredNodesBefore.get(SECONDARY)) ||
144                ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
145                  favoredNodesBefore.get(TERTIARY)));
146 
147     // put back the primary in the list of available servers
148     servers.addAll(removedServers);
149     // now roundRobinAssignment with the modified servers list should return the primary
150     // as the regionserver assignee
151     assignmentMap = balancer.roundRobinAssignment(regions, servers);
152     Set<ServerName> serverWithPrimary = assignmentMap.keySet();
153     assertTrue(serverBefore.containsAll(serverWithPrimary));
154 
155     // Make all the favored nodes unavailable for assignment
156     removeMatchingServers(favoredNodesAfter, servers);
157     // call roundRobinAssignment with the modified servers list
158     assignmentMap = balancer.roundRobinAssignment(regions, servers);
159     List<ServerName> favoredNodesNow =
160         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
161     assertTrue(favoredNodesNow.size() == 3);
162     assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
163         !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
164         !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
165   }
166 
167   @Test
168   public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException {
169     LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
170     balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
171     List<ServerName> servers = new ArrayList<ServerName>();
172     for (int i = 0; i < SLAVES; i++) {
173       ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
174       servers.add(server);
175     }
176     List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
177     HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
178     regions.add(region);
179     ServerName serverBefore = balancer.randomAssignment(region, servers);
180     List<ServerName> favoredNodesBefore =
181         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
182     assertTrue(favoredNodesBefore.size() == 3);
183     // the primary RS should be the one that the balancer's assignment returns
184     assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY)));
185     // now remove the primary from the list of servers
186     removeMatchingServers(serverBefore, servers);
187     // call randomAssignment with the modified servers list
188     ServerName serverAfter = balancer.randomAssignment(region, servers);
189     List<ServerName> favoredNodesAfter =
190         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
191     assertTrue(favoredNodesAfter.size() == 3);
192     // We don't expect the favored nodes assignments to change in multiple calls
193     // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign
194     // failures)
195     assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
196     // We expect the new RegionServer assignee to be one of the favored nodes
197     // chosen earlier.
198     assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY)) ||
199                ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY)));
200     // Make all the favored nodes unavailable for assignment
201     removeMatchingServers(favoredNodesAfter, servers);
202     // call randomAssignment with the modified servers list
203     balancer.randomAssignment(region, servers);
204     List<ServerName> favoredNodesNow =
205         ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
206     assertTrue(favoredNodesNow.size() == 3);
207     assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
208         !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
209         !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
210   }
211 
212   @Test
213   public void testRegionPlacement() throws Exception {
214     String tableStr = "testRegionAssignment";
215     byte[] table = Bytes.toBytes(tableStr);
216     // Create a table with REGION_NUM regions.
217     createTable(table, REGION_NUM);
218 
219     TEST_UTIL.waitTableAvailable(table);
220 
221     // Verify all the user regions are assigned to the primary region server
222     // based on the plan
223     verifyRegionOnPrimaryRS(REGION_NUM);
224 
225     FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
226     // Verify all the region server are update with the latest favored nodes
227     verifyRegionServerUpdated(currentPlan);
228     // Test Case 2: To verify whether the region placement tools can
229     // correctly update the new assignment plan to hbase:meta and Region Server.
230     // The new assignment plan is generated by shuffle the existing assignment
231     // plan by switching PRIMARY, SECONDARY and TERTIARY nodes.
232     // Shuffle the plan by switching the secondary region server with
233     // the tertiary.
234 
235     // Shuffle the secondary with tertiary favored nodes
236     FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
237         FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY);
238     // Let the region placement update the hbase:meta and Region Servers
239     rp.updateAssignmentPlan(shuffledPlan);
240 
241     // Verify the region assignment. There are supposed to no region reassignment
242     // All the regions are still on the primary region server
243     verifyRegionAssignment(shuffledPlan,0, REGION_NUM);
244 
245     // Shuffle the plan by switching the primary with secondary and
246     // verify the region reassignment is consistent with the plan.
247     shuffledPlan = this.shuffleAssignmentPlan(currentPlan,
248         FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY);
249 
250     // Let the region placement update the hbase:meta and Region Servers
251     rp.updateAssignmentPlan(shuffledPlan);
252 
253     verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM);
254 
255     // also verify that the AssignmentVerificationReport has the correct information
256     RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration());
257     // we are interested in only one table (and hence one report)
258     rp.setTargetTableName(new String[]{tableStr});
259     List<AssignmentVerificationReport> reports = rp.verifyRegionPlacement(false);
260     AssignmentVerificationReport report = reports.get(0);
261     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
262     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
263     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
264     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0);
265     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0);
266     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0);
267     assertTrue(report.getUnassignedRegions().size() == 0);
268 
269     // Check when a RS stops, the regions get assigned to their secondary/tertiary
270     killRandomServerAndVerifyAssignment();
271     
272     // also verify that the AssignmentVerificationReport has the correct information
273     reports = rp.verifyRegionPlacement(false);
274     report = reports.get(0);
275     assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0);
276     assertTrue(report.getNonFavoredAssignedRegions().size() == 0);
277     assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM);
278     assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0);
279     assertTrue("secondary " +
280     report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary "
281         + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY),
282         (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0
283         || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0));
284     assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) +
285         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) +
286         report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM);
287     RegionPlacementMaintainer.printAssignmentPlan(currentPlan);
288   }
289 
290   private void killRandomServerAndVerifyAssignment() 
291       throws IOException, InterruptedException, KeeperException {
292     ClusterStatus oldStatus = TEST_UTIL.getHBaseCluster().getClusterStatus();
293     ServerName servers[] = oldStatus.getServers().toArray(new ServerName[10]);
294     ServerName serverToKill = null;
295     int killIndex = 0;
296     Random random = new Random(System.currentTimeMillis());
297     ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta();
298     LOG.debug("Server holding meta " + metaServer);
299     boolean isNamespaceServer = false;
300     do {
301       // kill a random non-meta server carrying at least one region
302       killIndex = random.nextInt(servers.length);
303       serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName();
304       Collection<HRegion> regs =
305           TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext();
306       isNamespaceServer = false;
307       for (HRegion r : regs) {
308         if (r.getRegionInfo().getTable().getNamespaceAsString()
309             .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
310           isNamespaceServer = true;
311           break;
312         }
313       }
314     } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer ||
315         TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0);
316     LOG.debug("Stopping RS " + serverToKill);
317     Map<HRegionInfo, Pair<ServerName, ServerName>> regionsToVerify =
318         new HashMap<HRegionInfo, Pair<ServerName, ServerName>>();
319     // mark the regions to track
320     for (Map.Entry<HRegionInfo, ServerName[]> entry : favoredNodesAssignmentPlan.entrySet()) {
321       ServerName s = entry.getValue()[0];
322       if (ServerName.isSameHostnameAndPort(s, serverToKill)) {
323         regionsToVerify.put(entry.getKey(), new Pair<ServerName, ServerName>(
324             entry.getValue()[1], entry.getValue()[2]));
325         LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " +
326             entry.getValue()[1] + " " + entry.getValue()[2]);
327       }
328     }
329     int orig = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
330     TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill);
331     TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000);
332     int curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
333     while (curr - orig < regionsToVerify.size()) {
334       LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " +
335           " Current #regions " + curr + " Original #regions " + orig);
336       Thread.sleep(200);
337       curr = TEST_UTIL.getHBaseCluster().getMaster().assignmentManager.getNumRegionsOpened();
338     }
339     // now verify
340     for (Map.Entry<HRegionInfo, Pair<ServerName, ServerName>> entry : regionsToVerify.entrySet()) {
341       ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster()
342           .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey());
343       Pair<ServerName, ServerName> secondaryTertiaryServers = entry.getValue();
344       LOG.debug("New destination for region " + entry.getKey().getEncodedName() +
345           " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst()
346           + "/" + secondaryTertiaryServers.getSecond());
347       if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())||
348           ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){
349         fail("Region " + entry.getKey() + " not present on any of the expected servers");
350       }
351     }
352     // start(reinstate) region server since we killed one before
353     TEST_UTIL.getHBaseCluster().startRegionServer();
354   }
355 
356   /**
357    * Used to test the correctness of this class.
358    */
359   @Test
360   public void testRandomizedMatrix() {
361     int rows = 100;
362     int cols = 100;
363     float[][] matrix = new float[rows][cols];
364     Random random = new Random();
365     for (int i = 0; i < rows; i++) {
366       for (int j = 0; j < cols; j++) {
367         matrix[i][j] = random.nextFloat();
368       }
369     }
370 
371     // Test that inverting a transformed matrix gives the original matrix.
372     RegionPlacementMaintainer.RandomizedMatrix rm =
373       new RegionPlacementMaintainer.RandomizedMatrix(rows, cols);
374     float[][] transformed = rm.transform(matrix);
375     float[][] invertedTransformed = rm.invert(transformed);
376     for (int i = 0; i < rows; i++) {
377       for (int j = 0; j < cols; j++) {
378         if (matrix[i][j] != invertedTransformed[i][j]) {
379           throw new RuntimeException();
380         }
381       }
382     }
383 
384     // Test that the indices on a transformed matrix can be inverted to give
385     // the same values on the original matrix.
386     int[] transformedIndices = new int[rows];
387     for (int i = 0; i < rows; i++) {
388       transformedIndices[i] = random.nextInt(cols);
389     }
390     int[] invertedTransformedIndices = rm.invertIndices(transformedIndices);
391     float[] transformedValues = new float[rows];
392     float[] invertedTransformedValues = new float[rows];
393     for (int i = 0; i < rows; i++) {
394       transformedValues[i] = transformed[i][transformedIndices[i]];
395       invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]];
396     }
397     Arrays.sort(transformedValues);
398     Arrays.sort(invertedTransformedValues);
399     if (!Arrays.equals(transformedValues, invertedTransformedValues)) {
400       throw new RuntimeException();
401     }
402   }
403 
404   /**
405    * Shuffle the assignment plan by switching two favored node positions.
406    * @param plan The assignment plan
407    * @param p1 The first switch position
408    * @param p2 The second switch position
409    * @return
410    */
411   private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan,
412       FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) {
413     FavoredNodesPlan shuffledPlan = new FavoredNodesPlan();
414 
415     for (Map.Entry<HRegionInfo, List<ServerName>> entry :
416       plan.getAssignmentMap().entrySet()) {
417       HRegionInfo region = entry.getKey();
418 
419       // copy the server list from the original plan
420       List<ServerName> shuffledServerList = new ArrayList<ServerName>();
421       shuffledServerList.addAll(entry.getValue());
422 
423       // start to shuffle
424       shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal()));
425       shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal()));
426 
427       // update the plan
428       shuffledPlan.updateAssignmentPlan(region, shuffledServerList);
429     }
430     return shuffledPlan;
431   }
432 
433   /**
434    * To verify the region assignment status.
435    * It will check the assignment plan consistency between hbase:meta and
436    * region servers.
437    * Also it will verify weather the number of region movement and
438    * the number regions on the primary region server are expected
439    *
440    * @param plan
441    * @param regionMovementNum
442    * @param numRegionsOnPrimaryRS
443    * @throws InterruptedException
444    * @throws IOException
445    */
446   private void verifyRegionAssignment(FavoredNodesPlan plan,
447       int regionMovementNum, int numRegionsOnPrimaryRS)
448   throws InterruptedException, IOException {
449     // Verify the assignment plan in hbase:meta is consistent with the expected plan.
450     verifyMETAUpdated(plan);
451 
452     // Verify the number of region movement is expected
453     verifyRegionMovementNum(regionMovementNum);
454 
455     // Verify the number of regions is assigned to the primary region server
456     // based on the plan is expected
457     verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS);
458 
459     // Verify all the online region server are updated with the assignment plan
460     verifyRegionServerUpdated(plan);
461   }
462 
463   /**
464    * Verify the meta has updated to the latest assignment plan
465    * @param plan
466    * @throws IOException
467    */
468   private void verifyMETAUpdated(FavoredNodesPlan expectedPlan)
469   throws IOException {
470     FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan();
471     assertTrue("The assignment plan is NOT consistent with the expected plan ",
472         planFromMETA.equals(expectedPlan));
473   }
474 
475   /**
476    * Verify the number of region movement is expected
477    */
478   private void verifyRegionMovementNum(int expected)
479   throws InterruptedException, HBaseIOException {
480     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
481     HMaster m = cluster.getMaster();
482     int lastRegionOpenedCount = m.assignmentManager.getNumRegionsOpened();
483     // get the assignments start to execute
484     m.balance();
485 
486     int retry = 10;
487     long sleep = 3000;
488     int attempt = 0;
489     int currentRegionOpened, regionMovement;
490     do {
491       currentRegionOpened = m.assignmentManager.getNumRegionsOpened();
492       regionMovement= currentRegionOpened - lastRegionOpenedCount;
493       LOG.debug("There are " + regionMovement + "/" + expected +
494           " regions moved after " + attempt + " attempts");
495       Thread.sleep((++attempt) * sleep);
496     } while (regionMovement != expected && attempt <= retry);
497 
498     // update the lastRegionOpenedCount
499     lastRegionOpenedCount = currentRegionOpened;
500 
501     assertEquals("There are only " + regionMovement + " instead of "
502           + expected + " region movement for " + attempt + " attempts",
503           regionMovement, expected);
504   }
505 
506   private List<ServerName> removeMatchingServers(ServerName serverWithoutStartCode,
507       List<ServerName> servers) {
508     List<ServerName> serversToRemove = new ArrayList<ServerName>();
509     for (ServerName s : servers) {
510       if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) {
511         serversToRemove.add(s);
512       }
513     }
514     servers.removeAll(serversToRemove);
515     return serversToRemove;
516   }
517 
518   private List<ServerName> removeMatchingServers(Collection<ServerName> serversWithoutStartCode,
519       List<ServerName> servers) {
520     List<ServerName> serversToRemove = new ArrayList<ServerName>();
521     for (ServerName s : serversWithoutStartCode) {
522       serversToRemove.addAll(removeMatchingServers(s, servers));
523     }
524     return serversToRemove;
525   }
526 
527   /**
528    * Verify the number of user regions is assigned to the primary
529    * region server based on the plan is expected
530    * @param expectedNum.
531    * @throws IOException
532    */
533   private void verifyRegionOnPrimaryRS(int expectedNum)
534       throws IOException {
535     lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS();
536     assertEquals("Only " +  expectedNum + " of user regions running " +
537         "on the primary region server", expectedNum ,
538         lastRegionOnPrimaryRSCount);
539   }
540 
541   /**
542    * Verify all the online region servers has been updated to the
543    * latest assignment plan
544    * @param plan
545    * @throws IOException
546    */
547   private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException {
548     // Verify all region servers contain the correct favored nodes information
549     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
550     for (int i = 0; i < SLAVES; i++) {
551       HRegionServer rs = cluster.getRegionServer(i);
552       for (HRegion region: rs.getOnlineRegions(
553           TableName.valueOf("testRegionAssignment"))) {
554         InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion(
555             region.getRegionInfo().getEncodedName());
556         List<ServerName> favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo());
557 
558         // All regions are supposed to have favored nodes,
559         // except for hbase:meta and ROOT
560         if (favoredServerList == null) {
561           HTableDescriptor desc = region.getTableDesc();
562           // Verify they are ROOT and hbase:meta regions since no favored nodes
563           assertNull(favoredSocketAddress);
564           assertTrue("User region " +
565               region.getTableDesc().getTableName() +
566               " should have favored nodes",
567               (desc.isRootRegion() || desc.isMetaRegion()));
568         } else {
569           // For user region, the favored nodes in the region server should be
570           // identical to favored nodes in the assignmentPlan
571           assertTrue(favoredSocketAddress.length == favoredServerList.size());
572           assertTrue(favoredServerList.size() > 0);
573           for (int j = 0; j < favoredServerList.size(); j++) {
574             InetSocketAddress addrFromRS = favoredSocketAddress[j];
575             InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved(
576                 favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort());
577 
578             assertNotNull(addrFromRS);
579             assertNotNull(addrFromPlan);
580             assertTrue("Region server " + rs.getServerName().getHostAndPort()
581                 + " has the " + positions[j] +
582                 " for region " + region.getRegionNameAsString() + " is " +
583                 addrFromRS + " which is inconsistent with the plan "
584                 + addrFromPlan, addrFromRS.equals(addrFromPlan));
585           }
586         }
587       }
588     }
589   }
590 
591   /**
592    * Check whether regions are assigned to servers consistent with the explicit
593    * hints that are persisted in the hbase:meta table.
594    * Also keep track of the number of the regions are assigned to the
595    * primary region server.
596    * @return the number of regions are assigned to the primary region server
597    * @throws IOException
598    */
599   private int getNumRegionisOnPrimaryRS() throws IOException {
600     final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0);
601     final AtomicInteger totalRegionNum = new AtomicInteger(0);
602     LOG.info("The start of region placement verification");
603     MetaScannerVisitor visitor = new MetaScannerVisitor() {
604       public boolean processRow(Result result) throws IOException {
605         try {
606           HRegionInfo info = MetaScanner.getHRegionInfo(result);
607           if(info.getTable().getNamespaceAsString()
608               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
609             return true;
610           }
611           byte[] server = result.getValue(HConstants.CATALOG_FAMILY,
612               HConstants.SERVER_QUALIFIER);
613           byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
614               FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
615           // Add the favored nodes into assignment plan
616           ServerName[] favoredServerList =
617               FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
618           favoredNodesAssignmentPlan.put(info, favoredServerList);
619 
620           Position[] positions = Position.values();
621           if (info != null) {
622             totalRegionNum.incrementAndGet();
623             if (server != null) {
624               ServerName serverName =
625                   ServerName.valueOf(Bytes.toString(server), -1);
626               if (favoredNodes != null) {
627                 String placement = "[NOT FAVORED NODE]";
628                 for (int i = 0; i < favoredServerList.length; i++) {
629                   if (favoredServerList[i].equals(serverName)) {
630                     placement = positions[i].toString();
631                     if (i == Position.PRIMARY.ordinal()) {
632                       regionOnPrimaryNum.incrementAndGet();
633                     }
634                     break;
635                   }
636                 }
637                 LOG.info(info.getRegionNameAsString() + " on " +
638                     serverName + " " + placement);
639               } else {
640                 LOG.info(info.getRegionNameAsString() + " running on " +
641                     serverName + " but there is no favored region server");
642               }
643             } else {
644               LOG.info(info.getRegionNameAsString() +
645                   " not assigned to any server");
646             }
647           }
648           return true;
649         } catch (RuntimeException e) {
650           LOG.error("Result=" + result);
651           throw e;
652         }
653       }
654 
655       @Override
656       public void close() throws IOException {}
657     };
658     MetaScanner.metaScan(TEST_UTIL.getConfiguration(), visitor);
659     LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " +
660         totalRegionNum.intValue() + " regions running on the primary" +
661         " region servers" );
662     return regionOnPrimaryNum.intValue() ;
663   }
664 
665   /**
666    * Create a table with specified table name and region number.
667    * @param tablename
668    * @param regionNum
669    * @return
670    * @throws IOException
671    */
672   private static void createTable(byte[] tableName, int regionNum)
673       throws IOException {
674     int expectedRegions = regionNum;
675     byte[][] splitKeys = new byte[expectedRegions - 1][];
676     for (int i = 1; i < expectedRegions; i++) {
677       byte splitKey = (byte) i;
678       splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
679     }
680 
681     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
682     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
683     admin.createTable(desc, splitKeys);
684 
685     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
686     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
687     assertEquals("Tried to create " + expectedRegions + " regions "
688         + "but only found " + regions.size(), expectedRegions, regions.size());
689   }
690 }