View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.hbase.classification.InterfaceAudience;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.ServerName;
28  import org.apache.hadoop.hbase.client.Get;
29  import org.apache.hadoop.hbase.client.HTable;
30  import org.apache.hadoop.hbase.client.Result;
31  import org.apache.hadoop.hbase.client.ResultScanner;
32  import org.apache.hadoop.hbase.client.Scan;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.Pair;
35  
36  import java.io.IOException;
37  import java.io.InterruptedIOException;
38  import java.util.ArrayList;
39  import java.util.List;
40  import java.util.Map;
41  import java.util.NavigableMap;
42  import java.util.Set;
43  import java.util.TreeMap;
44  
45  /**
46   * Reads region and assignment information from <code>hbase:meta</code>.
47   */
48  @InterfaceAudience.Private
49  public class MetaReader {
50    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
51    // its only used to get its Configuration so we can get associated
52    // Connection.
53    private static final Log LOG = LogFactory.getLog(MetaReader.class);
54  
55    static final byte [] META_REGION_PREFIX;
56    static {
57      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
58      // FIRST_META_REGIONINFO == 'hbase:meta,,1'.  META_REGION_PREFIX == 'hbase:meta,'
59      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
60      META_REGION_PREFIX = new byte [len];
61      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
62        META_REGION_PREFIX, 0, len);
63    }
64  
65    /**
66     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
67     * tables in the specified set of disabled tables.
68     * @param catalogTracker
69     * @param disabledTables set of disabled tables that will not be returned
70     * @return Returns a map of every region to it's currently assigned server,
71     * according to META.  If the region does not have an assignment it will have
72     * a null value in the map.
73     * @throws IOException
74     */
75    public static Map<HRegionInfo, ServerName> fullScan(
76        CatalogTracker catalogTracker, final Set<TableName> disabledTables)
77    throws IOException {
78      return fullScan(catalogTracker, disabledTables, false);
79    }
80  
81    /**
82     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
83     * tables in the specified set of disabled tables.
84     * @param catalogTracker
85     * @param disabledTables set of disabled tables that will not be returned
86     * @param excludeOfflinedSplitParents If true, do not include offlined split
87     * parents in the return.
88     * @return Returns a map of every region to it's currently assigned server,
89     * according to META.  If the region does not have an assignment it will have
90     * a null value in the map.
91     * @throws IOException
92     */
93    public static Map<HRegionInfo, ServerName> fullScan(
94        CatalogTracker catalogTracker, final Set<TableName> disabledTables,
95        final boolean excludeOfflinedSplitParents)
96    throws IOException {
97      final Map<HRegionInfo, ServerName> regions =
98        new TreeMap<HRegionInfo, ServerName>();
99      Visitor v = new Visitor() {
100       @Override
101       public boolean visit(Result r) throws IOException {
102         if (r ==  null || r.isEmpty()) return true;
103         Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
104         HRegionInfo hri = region.getFirst();
105         if (hri  == null) return true;
106         if (hri.getTable() == null) return true;
107         if (disabledTables.contains(
108             hri.getTable())) return true;
109         // Are we to include split parents in the list?
110         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
111         regions.put(hri, region.getSecond());
112         return true;
113       }
114     };
115     fullScan(catalogTracker, v);
116     return regions;
117   }
118 
119   /**
120    * Performs a full scan of <code>hbase:meta</code>.
121    * @return List of {@link Result}
122    * @throws IOException
123    */
124   public static List<Result> fullScan(CatalogTracker catalogTracker)
125   throws IOException {
126     CollectAllVisitor v = new CollectAllVisitor();
127     fullScan(catalogTracker, v, null);
128     return v.getResults();
129   }
130 
131   /**
132    * Performs a full scan of a <code>hbase:meta</code> table.
133    * @return List of {@link Result}
134    * @throws IOException
135    */
136   public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
137   throws IOException {
138     CollectAllVisitor v = new CollectAllVisitor();
139     fullScan(catalogTracker, v, null);
140     return v.getResults();
141   }
142 
143   /**
144    * Performs a full scan of <code>hbase:meta</code>.
145    * @param catalogTracker
146    * @param visitor Visitor invoked against each row.
147    * @throws IOException
148    */
149   public static void fullScan(CatalogTracker catalogTracker,
150       final Visitor visitor)
151   throws IOException {
152     fullScan(catalogTracker, visitor, null);
153   }
154 
155   /**
156    * Callers should call close on the returned {@link HTable} instance.
157    * @param catalogTracker We'll use this catalogtracker's connection
158    * @param tableName Table to get an {@link HTable} against.
159    * @return An {@link HTable} for <code>tableName</code>
160    * @throws IOException
161    */
162   @SuppressWarnings("deprecation")
163   private static HTable getHTable(final CatalogTracker catalogTracker,
164       final TableName tableName)
165   throws IOException {
166     // Passing the CatalogTracker's connection ensures this
167     // HTable instance uses the CatalogTracker's connection.
168     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
169     if (c == null) throw new NullPointerException("No connection");
170     return new HTable(tableName, c);
171   }
172 
173   /**
174    * Callers should call close on the returned {@link HTable} instance.
175    * @param catalogTracker
176    * @return An {@link HTable} for <code>hbase:meta</code>
177    * @throws IOException
178    */
179   static HTable getCatalogHTable(final CatalogTracker catalogTracker)
180   throws IOException {
181     return getMetaHTable(catalogTracker);
182   }
183 
184   /**
185    * Callers should call close on the returned {@link HTable} instance.
186    * @param ct
187    * @return An {@link HTable} for <code>hbase:meta</code>
188    * @throws IOException
189    */
190   static HTable getMetaHTable(final CatalogTracker ct)
191   throws IOException {
192     return getHTable(ct, TableName.META_TABLE_NAME);
193   }
194 
195   /**
196    * @param t Table to use (will be closed when done).
197    * @param g Get to run
198    * @throws IOException
199    */
200   private static Result get(final HTable t, final Get g) throws IOException {
201     try {
202       return t.get(g);
203     } finally {
204       t.close();
205     }
206   }
207 
208   /**
209    * Reads the location of the specified region
210    * @param catalogTracker
211    * @param regionName region whose location we are after
212    * @return location of region as a {@link ServerName} or null if not found
213    * @throws IOException
214    */
215   static ServerName readRegionLocation(CatalogTracker catalogTracker,
216       byte [] regionName)
217   throws IOException {
218     Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
219     return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
220   }
221 
222   /**
223    * Gets the region info and assignment for the specified region.
224    * @param catalogTracker
225    * @param regionName Region to lookup.
226    * @return Location and HRegionInfo for <code>regionName</code>
227    * @throws IOException
228    */
229   public static Pair<HRegionInfo, ServerName> getRegion(
230       CatalogTracker catalogTracker, byte [] regionName)
231   throws IOException {
232     Get get = new Get(regionName);
233     get.addFamily(HConstants.CATALOG_FAMILY);
234     Result r = get(getCatalogHTable(catalogTracker), get);
235     return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
236   }
237 
238   /**
239    * Gets the result in hbase:meta for the specified region.
240    * @param catalogTracker
241    * @param regionName
242    * @return result of the specified region
243    * @throws IOException
244    */
245   public static Result getRegionResult(CatalogTracker catalogTracker,
246       byte[] regionName) throws IOException {
247     Get get = new Get(regionName);
248     get.addFamily(HConstants.CATALOG_FAMILY);
249     return get(getCatalogHTable(catalogTracker), get);
250   }
251 
252   /**
253    * Get regions from the merge qualifier of the specified merged region
254    * @return null if it doesn't contain merge qualifier, else two merge regions
255    * @throws IOException
256    */
257   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
258       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
259     Result result = getRegionResult(catalogTracker, regionName);
260     HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
261         HConstants.MERGEA_QUALIFIER);
262     HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
263         HConstants.MERGEB_QUALIFIER);
264     if (mergeA == null && mergeB == null) {
265       return null;
266     }
267     return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
268  }
269 
270   /**
271    * Checks if the specified table exists.  Looks at the hbase:meta table hosted on
272    * the specified server.
273    * @param catalogTracker
274    * @param tableName table to check
275    * @return true if the table exists in meta, false if not
276    * @throws IOException
277    */
278   public static boolean tableExists(CatalogTracker catalogTracker,
279       final TableName tableName)
280   throws IOException {
281     if (tableName.equals(TableName.META_TABLE_NAME)) {
282       // Catalog tables always exist.
283       return true;
284     }
285     // Make a version of ResultCollectingVisitor that only collects the first
286     CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
287       private HRegionInfo current = null;
288 
289       @Override
290       public boolean visit(Result r) throws IOException {
291         this.current =
292           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
293         if (this.current == null) {
294           LOG.warn("No serialized HRegionInfo in " + r);
295           return true;
296         }
297         if (!isInsideTable(this.current, tableName)) return false;
298         // Else call super and add this Result to the collection.
299         super.visit(r);
300         // Stop collecting regions from table after we get one.
301         return false;
302       }
303 
304       @Override
305       void add(Result r) {
306         // Add the current HRI.
307         this.results.add(this.current);
308       }
309     };
310     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
311     // If visitor has results >= 1 then table exists.
312     return visitor.getResults().size() >= 1;
313   }
314 
315   /**
316    * Gets all of the regions of the specified table.
317    * @param catalogTracker
318    * @param tableName
319    * @return Ordered list of {@link HRegionInfo}.
320    * @throws IOException
321    */
322   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
323       TableName tableName)
324   throws IOException {
325     return getTableRegions(catalogTracker, tableName, false);
326   }
327 
328   /**
329    * Gets all of the regions of the specified table.
330    * @param catalogTracker
331    * @param tableName
332    * @param excludeOfflinedSplitParents If true, do not include offlined split
333    * parents in the return.
334    * @return Ordered list of {@link HRegionInfo}.
335    * @throws IOException
336    */
337   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
338       TableName tableName, final boolean excludeOfflinedSplitParents)
339   throws IOException {
340     List<Pair<HRegionInfo, ServerName>> result = null;
341     try {
342       result = getTableRegionsAndLocations(catalogTracker, tableName,
343         excludeOfflinedSplitParents);
344     } catch (InterruptedException e) {
345       throw (InterruptedIOException)new InterruptedIOException().initCause(e);
346     }
347     return getListOfHRegionInfos(result);
348   }
349 
350   static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
351     if (pairs == null || pairs.isEmpty()) return null;
352     List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
353     for (Pair<HRegionInfo, ServerName> pair: pairs) {
354       result.add(pair.getFirst());
355     }
356     return result;
357   }
358 
359   /**
360    * @param current
361    * @param tableName
362    * @return True if <code>current</code> tablename is equal to
363    * <code>tableName</code>
364    */
365   static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
366     return tableName.equals(current.getTable());
367   }
368 
369   /**
370    * @param tableName
371    * @return Place to start Scan in <code>hbase:meta</code> when passed a
372    * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
373    */
374   static byte [] getTableStartRowForMeta(TableName tableName) {
375     byte [] startRow = new byte[tableName.getName().length + 2];
376     System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
377     startRow[startRow.length - 2] = HConstants.DELIMITER;
378     startRow[startRow.length - 1] = HConstants.DELIMITER;
379     return startRow;
380   }
381 
382   /**
383    * This method creates a Scan object that will only scan catalog rows that
384    * belong to the specified table. It doesn't specify any columns.
385    * This is a better alternative to just using a start row and scan until
386    * it hits a new table since that requires parsing the HRI to get the table
387    * name.
388    * @param tableName bytes of table's name
389    * @return configured Scan object
390    */
391   public static Scan getScanForTableName(TableName tableName) {
392     String strName = tableName.getNameAsString();
393     // Start key is just the table name with delimiters
394     byte[] startKey = Bytes.toBytes(strName + ",,");
395     // Stop key appends the smallest possible char to the table name
396     byte[] stopKey = Bytes.toBytes(strName + " ,,");
397 
398     Scan scan = new Scan(startKey);
399     scan.setStopRow(stopKey);
400     return scan;
401   }
402 
403   /**
404    * @param catalogTracker
405    * @param tableName
406    * @return Return list of regioninfos and server.
407    * @throws IOException
408    * @throws InterruptedException
409    */
410   public static List<Pair<HRegionInfo, ServerName>>
411   getTableRegionsAndLocations(CatalogTracker catalogTracker, TableName tableName)
412   throws IOException, InterruptedException {
413     return getTableRegionsAndLocations(catalogTracker, tableName,
414       true);
415   }
416 
417   /**
418    * @param catalogTracker
419    * @param tableName
420    * @return Return list of regioninfos and server addresses.
421    * @throws IOException
422    * @throws InterruptedException
423    */
424   public static List<Pair<HRegionInfo, ServerName>>
425   getTableRegionsAndLocations(final CatalogTracker catalogTracker,
426       final TableName tableName, final boolean excludeOfflinedSplitParents)
427   throws IOException, InterruptedException {
428     if (tableName.equals(TableName.META_TABLE_NAME)) {
429       // If meta, do a bit of special handling.
430       ServerName serverName = catalogTracker.getMetaLocation();
431       List<Pair<HRegionInfo, ServerName>> list =
432           new ArrayList<Pair<HRegionInfo, ServerName>>();
433       list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
434           serverName));
435       return list;
436     }
437     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
438     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
439         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
440       private Pair<HRegionInfo, ServerName> current = null;
441 
442       @Override
443       public boolean visit(Result r) throws IOException {
444         HRegionInfo hri =
445           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
446         if (hri == null) {
447           LOG.warn("No serialized HRegionInfo in " + r);
448           return true;
449         }
450         if (!isInsideTable(hri, tableName)) return false;
451         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
452         ServerName sn = HRegionInfo.getServerName(r);
453         // Populate this.current so available when we call #add
454         this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
455         // Else call super and add this Result to the collection.
456         return super.visit(r);
457       }
458 
459       @Override
460       void add(Result r) {
461         this.results.add(this.current);
462       }
463     };
464     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
465     return visitor.getResults();
466   }
467 
468   /**
469    * @param catalogTracker
470    * @param serverName
471    * @return List of user regions installed on this server (does not include
472    * catalog regions).
473    * @throws IOException
474    */
475   public static NavigableMap<HRegionInfo, Result>
476   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
477   throws IOException {
478     final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
479     // Fill the above hris map with entries from hbase:meta that have the passed
480     // servername.
481     CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
482       @Override
483       void add(Result r) {
484         if (r == null || r.isEmpty()) return;
485         if (HRegionInfo.getHRegionInfo(r) == null) return;
486         ServerName sn = HRegionInfo.getServerName(r);
487         if (sn != null && sn.equals(serverName)) {
488           this.results.add(r);
489         }
490       }
491     };
492     fullScan(catalogTracker, v);
493     List<Result> results = v.getResults();
494     if (results != null && !results.isEmpty()) {
495       // Convert results to Map keyed by HRI
496       for (Result r: results) {
497         HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
498         if (hri != null) hris.put(hri, r);
499       }
500     }
501     return hris;
502   }
503 
504   public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
505   throws IOException {
506     Visitor v = new Visitor() {
507       @Override
508       public boolean visit(Result r) throws IOException {
509         if (r ==  null || r.isEmpty()) return true;
510         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
511         HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
512         LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
513         return true;
514       }
515     };
516     fullScan(catalogTracker, v);
517   }
518 
519   /**
520    * Performs a full scan of a catalog table.
521    * @param catalogTracker
522    * @param visitor Visitor invoked against each row.
523    * @param startrow Where to start the scan. Pass null if want to begin scan
524    * at first row.
525    * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
526    * @throws IOException
527    */
528   public static void fullScan(CatalogTracker catalogTracker,
529     final Visitor visitor, final byte [] startrow)
530   throws IOException {
531     Scan scan = new Scan();
532     if (startrow != null) scan.setStartRow(startrow);
533     if (startrow == null) {
534       int caching = catalogTracker.getConnection().getConfiguration()
535           .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
536       scan.setCaching(caching);
537     }
538     scan.addFamily(HConstants.CATALOG_FAMILY);
539     HTable metaTable = getMetaHTable(catalogTracker);
540     ResultScanner scanner = null;
541     try {
542       scanner = metaTable.getScanner(scan);
543       Result data;
544       while((data = scanner.next()) != null) {
545         if (data.isEmpty()) continue;
546         // Break if visit returns false.
547         if (!visitor.visit(data)) break;
548       }
549     } finally {
550       if (scanner != null) scanner.close();
551       metaTable.close();
552     }
553     return;
554   }
555 
556   /**
557    * Implementations 'visit' a catalog table row.
558    */
559   public interface Visitor {
560     /**
561      * Visit the catalog table row.
562      * @param r A row from catalog table
563      * @return True if we are to proceed scanning the table, else false if
564      * we are to stop now.
565      */
566     boolean visit(final Result r) throws IOException;
567   }
568 
569   /**
570    * A {@link Visitor} that collects content out of passed {@link Result}.
571    */
572   static abstract class CollectingVisitor<T> implements Visitor {
573     final List<T> results = new ArrayList<T>();
574     @Override
575     public boolean visit(Result r) throws IOException {
576       if (r ==  null || r.isEmpty()) return true;
577       add(r);
578       return true;
579     }
580 
581     abstract void add(Result r);
582 
583     /**
584      * @return Collected results; wait till visits complete to collect all
585      * possible results
586      */
587     List<T> getResults() {
588       return this.results;
589     }
590   }
591 
592   /**
593    * Collects all returned.
594    */
595   static class CollectAllVisitor extends CollectingVisitor<Result> {
596     @Override
597     void add(Result r) {
598       this.results.add(r);
599     }
600   }
601 
602   /**
603    * Count regions in <code>hbase:meta</code> for passed table.
604    * @param c
605    * @param tableName
606    * @return Count or regions in table <code>tableName</code>
607    * @throws IOException
608    */
609   public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
610     HTable t = new HTable(c, tableName);
611     try {
612       return t.getRegionLocations().size();
613     } finally {
614       t.close();
615     }
616   }
617 }