View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.NavigableSet;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.hbase.classification.InterfaceAudience;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileStatus;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.hbase.CellUtil;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.NamespaceDescriptor;
37  import org.apache.hadoop.hbase.NamespaceExistException;
38  import org.apache.hadoop.hbase.NamespaceNotFoundException;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.ZKNamespaceManager;
41  import org.apache.hadoop.hbase.catalog.MetaReader;
42  import org.apache.hadoop.hbase.client.Delete;
43  import org.apache.hadoop.hbase.client.Get;
44  import org.apache.hadoop.hbase.client.HTable;
45  import org.apache.hadoop.hbase.client.Put;
46  import org.apache.hadoop.hbase.client.Result;
47  import org.apache.hadoop.hbase.client.ResultScanner;
48  import org.apache.hadoop.hbase.constraint.ConstraintException;
49  import org.apache.hadoop.hbase.master.handler.CreateTableHandler;
50  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
51  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
54  import org.apache.hadoop.hbase.util.FSUtils;
55  
56  import com.google.common.collect.Sets;
57  
58  /**
59   * This is a helper class used to manage the namespace
60   * metadata that is stored in TableName.NAMESPACE_TABLE_NAME
61   * It also mirrors updates to the ZK store by forwarding updates to
62   * {@link org.apache.hadoop.hbase.ZKNamespaceManager}
63   */
64  @InterfaceAudience.Private
65  public class TableNamespaceManager {
66    private static final Log LOG = LogFactory.getLog(TableNamespaceManager.class);
67  
68    private Configuration conf;
69    private MasterServices masterServices;
70    private HTable nsTable;
71    private ZKNamespaceManager zkNamespaceManager;
72    private boolean initialized;
73  
74    static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout";
75    static final int DEFAULT_NS_INIT_TIMEOUT = 60000;
76  
77    public TableNamespaceManager(MasterServices masterServices) {
78      this.masterServices = masterServices;
79      this.conf = masterServices.getConfiguration();
80    }
81  
82    public void start() throws IOException {
83      if (!MetaReader.tableExists(masterServices.getCatalogTracker(),
84          TableName.NAMESPACE_TABLE_NAME)) {
85        LOG.info("Namespace table not found. Creating...");
86        createNamespaceTable(masterServices);
87      }
88  
89      try {
90        // Wait for the namespace table to be assigned.
91        // If timed out, we will move ahead without initializing it.
92        // So that it should be initialized later on lazily.
93        long startTime = EnvironmentEdgeManager.currentTimeMillis();
94        int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT);
95        while (!isTableAssigned()) {
96          if (EnvironmentEdgeManager.currentTimeMillis() - startTime + 100 > timeout) {
97            LOG.warn("Timedout waiting for namespace table to be assigned.");
98            return;
99          }
100         Thread.sleep(100);
101       }
102     } catch (InterruptedException e) {
103       throw (InterruptedIOException)new InterruptedIOException().initCause(e);
104     }
105 
106     // initialize namespace table
107     isTableAvailableAndInitialized();
108   }
109 
110   private synchronized HTable getNamespaceTable() throws IOException {
111     if (!isTableAvailableAndInitialized()) {
112       throw new IOException(this.getClass().getName() + " isn't ready to serve");
113     }
114     return nsTable;
115   }
116 
117 
118   public synchronized NamespaceDescriptor get(String name) throws IOException {
119     if (!isTableAvailableAndInitialized()) return null;
120     return zkNamespaceManager.get(name);
121   }
122 
123   public synchronized void create(NamespaceDescriptor ns) throws IOException {
124     create(getNamespaceTable(), ns);
125   }
126 
127   public synchronized void update(NamespaceDescriptor ns) throws IOException {
128     HTable table = getNamespaceTable();
129     if (get(table, ns.getName()) == null) {
130       throw new NamespaceNotFoundException(ns.getName());
131     }
132     upsert(table, ns);
133   }
134 
135   private NamespaceDescriptor get(HTable table, String name) throws IOException {
136     Result res = table.get(new Get(Bytes.toBytes(name)));
137     if (res.isEmpty()) {
138       return null;
139     }
140     byte[] val = CellUtil.cloneValue(res.getColumnLatestCell(
141         HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
142     return
143         ProtobufUtil.toNamespaceDescriptor(
144             HBaseProtos.NamespaceDescriptor.parseFrom(val));
145   }
146 
147   private void create(HTable table, NamespaceDescriptor ns) throws IOException {
148     if (get(table, ns.getName()) != null) {
149       throw new NamespaceExistException(ns.getName());
150     }
151     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
152     fs.mkdirs(FSUtils.getNamespaceDir(
153         masterServices.getMasterFileSystem().getRootDir(), ns.getName()));
154     upsert(table, ns);
155   }
156 
157   private void upsert(HTable table, NamespaceDescriptor ns) throws IOException {
158     Put p = new Put(Bytes.toBytes(ns.getName()));
159     p.addImmutable(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
160         HTableDescriptor.NAMESPACE_COL_DESC_BYTES,
161         ProtobufUtil.toProtoNamespaceDescriptor(ns).toByteArray());
162     table.put(p);
163     try {
164       zkNamespaceManager.update(ns);
165     } catch(IOException ex) {
166       String msg = "Failed to update namespace information in ZK. Aborting.";
167       LOG.fatal(msg, ex);
168       masterServices.abort(msg, ex);
169     }
170   }
171 
172   public synchronized void remove(String name) throws IOException {
173     if (get(name) == null) {
174       throw new NamespaceNotFoundException(name);
175     }
176     if (NamespaceDescriptor.RESERVED_NAMESPACES.contains(name)) {
177       throw new ConstraintException("Reserved namespace "+name+" cannot be removed.");
178     }
179     int tableCount;
180     try {
181       tableCount = masterServices.listTableDescriptorsByNamespace(name).size();
182     } catch (FileNotFoundException fnfe) {
183       throw new NamespaceNotFoundException(name);
184     }
185     if (tableCount > 0) {
186       throw new ConstraintException("Only empty namespaces can be removed. " +
187           "Namespace "+name+" has "+tableCount+" tables");
188     }
189     Delete d = new Delete(Bytes.toBytes(name));
190     getNamespaceTable().delete(d);
191     //don't abort if cleanup isn't complete
192     //it will be replaced on new namespace creation
193     zkNamespaceManager.remove(name);
194     FileSystem fs = masterServices.getMasterFileSystem().getFileSystem();
195     for(FileStatus status :
196             fs.listStatus(FSUtils.getNamespaceDir(
197                 masterServices.getMasterFileSystem().getRootDir(), name))) {
198       if (!HConstants.HBASE_NON_TABLE_DIRS.contains(status.getPath().getName())) {
199         throw new IOException("Namespace directory contains table dir: "+status.getPath());
200       }
201     }
202     if (!fs.delete(FSUtils.getNamespaceDir(
203         masterServices.getMasterFileSystem().getRootDir(), name), true)) {
204       throw new IOException("Failed to remove namespace: "+name);
205     }
206   }
207 
208   public synchronized NavigableSet<NamespaceDescriptor> list() throws IOException {
209     NavigableSet<NamespaceDescriptor> ret =
210         Sets.newTreeSet(NamespaceDescriptor.NAMESPACE_DESCRIPTOR_COMPARATOR);
211     ResultScanner scanner = getNamespaceTable().getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
212     try {
213       for(Result r : scanner) {
214         byte[] val = CellUtil.cloneValue(r.getColumnLatestCell(
215           HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
216           HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
217         ret.add(ProtobufUtil.toNamespaceDescriptor(
218             HBaseProtos.NamespaceDescriptor.parseFrom(val)));
219       }
220     } finally {
221       scanner.close();
222     }
223     return ret;
224   }
225 
226   private void createNamespaceTable(MasterServices masterServices) throws IOException {
227     HRegionInfo newRegions[] = new HRegionInfo[]{
228         new HRegionInfo(HTableDescriptor.NAMESPACE_TABLEDESC.getTableName(), null, null)};
229 
230     //we need to create the table this way to bypass
231     //checkInitialized
232     masterServices.getExecutorService()
233         .submit(new CreateTableHandler(masterServices,
234             masterServices.getMasterFileSystem(),
235             HTableDescriptor.NAMESPACE_TABLEDESC,
236             masterServices.getConfiguration(),
237             newRegions,
238             masterServices).prepare());
239   }
240 
241   /**
242    * This method checks if the namespace table is assigned and then
243    * tries to create its HTable. If it was already created before, it also makes
244    * sure that the connection isn't closed.
245    * @return true if the namespace table manager is ready to serve, false
246    * otherwise
247    * @throws IOException
248    */
249   @SuppressWarnings("deprecation")
250   public synchronized boolean isTableAvailableAndInitialized() throws IOException {
251     // Did we already get a table? If so, still make sure it's available
252     if (initialized) {
253       if (nsTable.getConnection().isClosed()) {
254         nsTable = new HTable(conf, TableName.NAMESPACE_TABLE_NAME);
255       }
256       return true;
257     }
258 
259     // Now check if the table is assigned, if not then fail fast
260     if (isTableAssigned()) {
261       try {
262         nsTable = new HTable(conf, TableName.NAMESPACE_TABLE_NAME);
263         zkNamespaceManager = new ZKNamespaceManager(masterServices.getZooKeeper());
264         zkNamespaceManager.start();
265 
266         if (get(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE.getName()) == null) {
267           create(nsTable, NamespaceDescriptor.DEFAULT_NAMESPACE);
268         }
269         if (get(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE.getName()) == null) {
270           create(nsTable, NamespaceDescriptor.SYSTEM_NAMESPACE);
271         }
272 
273         ResultScanner scanner = nsTable.getScanner(HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES);
274         try {
275           for (Result result : scanner) {
276             byte[] val =  CellUtil.cloneValue(result.getColumnLatest(
277                 HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES,
278                 HTableDescriptor.NAMESPACE_COL_DESC_BYTES));
279             NamespaceDescriptor ns =
280                 ProtobufUtil.toNamespaceDescriptor(
281                     HBaseProtos.NamespaceDescriptor.parseFrom(val));
282             zkNamespaceManager.update(ns);
283           }
284         } finally {
285           scanner.close();
286         }
287         initialized = true;
288         return true;
289       } catch (IOException ie) {
290         LOG.warn("Caught exception in initializing namespace table manager", ie);
291         if (nsTable != null) {
292           nsTable.close();
293         }
294         throw ie;
295       }
296     }
297     return false;
298   }
299 
300   private boolean isTableAssigned() {
301     return !masterServices.getAssignmentManager()
302         .getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME).isEmpty();
303   }
304 }