View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import com.sun.jersey.api.client.Client;
22  import com.sun.jersey.api.client.ClientResponse;
23  import com.sun.jersey.api.client.filter.HTTPBasicAuthFilter;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.conf.Configured;
29  import org.apache.hadoop.util.ReflectionUtils;
30  import org.codehaus.jackson.JsonNode;
31  import org.codehaus.jackson.map.ObjectMapper;
32  
33  import javax.ws.rs.core.MediaType;
34  import javax.ws.rs.core.Response;
35  import javax.ws.rs.core.UriBuilder;
36  import javax.xml.ws.http.HTTPException;
37  import java.io.IOException;
38  import java.net.URI;
39  import java.util.HashMap;
40  import java.util.Map;
41  
42  /**
43   * A ClusterManager implementation designed to control Cloudera Manager (http://www.cloudera.com)
44   * clusters via REST API. This API uses HTTP GET requests against the cluster manager server to
45   * retrieve information and POST/PUT requests to perform actions. As a simple example, to retrieve a
46   * list of hosts from a CM server with login credentials admin:admin, a simple curl command would be
47   *     curl -X POST -H "Content-Type:application/json" -u admin:admin \
48   *         "http://this.is.my.server.com:7180/api/v8/hosts"
49   *
50   * This command would return a JSON result, which would need to be parsed to retrieve relevant
51   * information. This action and many others are covered by this class.
52   *
53   * A note on nomenclature: while the ClusterManager interface uses a ServiceType enum when
54   * referring to things like RegionServers and DataNodes, cluster managers often use different
55   * terminology. As an example, Cloudera Manager (http://www.cloudera.com) would refer to a
56   * RegionServer as a "role" of the HBase "service." It would further refer to "hbase" as the
57   * "serviceType." Apache Ambari (http://ambari.apache.org) would call the RegionServer a
58   * "component" of the HBase "service."
59   *
60   * This class will defer to the ClusterManager terminology in methods that it implements from
61   * that interface, but uses Cloudera Manager's terminology when dealing with its API directly.
62   */
63  public class RESTApiClusterManager extends Configured implements ClusterManager {
64    // Properties that need to be in the Configuration object to interact with the REST API cluster
65    // manager. Most easily defined in hbase-site.xml, but can also be passed on the command line.
66    private static final String REST_API_CLUSTER_MANAGER_HOSTNAME =
67        "hbase.it.clustermanager.restapi.hostname";
68    private static final String REST_API_CLUSTER_MANAGER_USERNAME =
69        "hbase.it.clustermanager.restapi.username";
70    private static final String REST_API_CLUSTER_MANAGER_PASSWORD =
71        "hbase.it.clustermanager.restapi.password";
72    private static final String REST_API_CLUSTER_MANAGER_CLUSTER_NAME =
73        "hbase.it.clustermanager.restapi.clustername";
74  
75    // Some default values for the above properties.
76    private static final String DEFAULT_SERVER_HOSTNAME = "http://localhost:7180";
77    private static final String DEFAULT_SERVER_USERNAME = "admin";
78    private static final String DEFAULT_SERVER_PASSWORD = "admin";
79    private static final String DEFAULT_CLUSTER_NAME = "Cluster 1";
80  
81    // Fields for the hostname, username, password, and cluster name of the cluster management server
82    // to be used.
83    private String serverHostname;
84    private String serverUsername;
85    private String serverPassword;
86    private String clusterName;
87  
88    // Each version of Cloudera Manager supports a particular API versions. Version 6 of this API
89    // provides all the features needed by this class.
90    private static final String API_VERSION = "v6";
91  
92    // Client instances are expensive, so use the same one for all our REST queries.
93    private Client client = Client.create();
94  
95    // An instance of HBaseClusterManager is used for methods like the kill, resume, and suspend
96    // because cluster managers don't tend to implement these operations.
97    private ClusterManager hBaseClusterManager;
98  
99    private static final Log LOG = LogFactory.getLog(RESTApiClusterManager.class);
100 
101   RESTApiClusterManager() {
102     hBaseClusterManager = ReflectionUtils.newInstance(HBaseClusterManager.class,
103         new IntegrationTestingUtility().getConfiguration());
104   }
105 
106   @Override
107   public void setConf(Configuration conf) {
108     super.setConf(conf);
109     if (conf == null) {
110       // Configured gets passed null before real conf. Why? I don't know.
111       return;
112     }
113     serverHostname = conf.get(REST_API_CLUSTER_MANAGER_HOSTNAME, DEFAULT_SERVER_HOSTNAME);
114     serverUsername = conf.get(REST_API_CLUSTER_MANAGER_USERNAME, DEFAULT_SERVER_USERNAME);
115     serverPassword = conf.get(REST_API_CLUSTER_MANAGER_PASSWORD, DEFAULT_SERVER_PASSWORD);
116     clusterName = conf.get(REST_API_CLUSTER_MANAGER_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
117 
118     // Add filter to Client instance to enable server authentication.
119     client.addFilter(new HTTPBasicAuthFilter(serverUsername, serverPassword));
120   }
121 
122   @Override
123   public void start(ServiceType service, String hostname, int port) throws IOException {
124     performClusterManagerCommand(service, hostname, RoleCommand.START);
125   }
126 
127   @Override
128   public void stop(ServiceType service, String hostname, int port) throws IOException {
129     performClusterManagerCommand(service, hostname, RoleCommand.STOP);
130   }
131 
132   @Override
133   public void restart(ServiceType service, String hostname, int port) throws IOException {
134     performClusterManagerCommand(service, hostname, RoleCommand.RESTART);
135   }
136 
137   @Override
138   public boolean isRunning(ServiceType service, String hostname, int port) throws IOException {
139     String serviceName = getServiceName(roleServiceType.get(service));
140     String hostId = getHostId(hostname);
141     String roleState = getRoleState(serviceName, service.toString(), hostId);
142     String healthSummary = getHealthSummary(serviceName, service.toString(), hostId);
143     boolean isRunning = false;
144 
145     // Use Yoda condition to prevent NullPointerException. roleState will be null if the "service
146     // type" does not exist on the specified hostname.
147     if ("STARTED".equals(roleState) && "GOOD".equals(healthSummary)) {
148       isRunning = true;
149     }
150 
151     return isRunning;
152   }
153 
154   @Override
155   public void kill(ServiceType service, String hostname, int port) throws IOException {
156     hBaseClusterManager.kill(service, hostname, port);
157   }
158 
159   @Override
160   public void suspend(ServiceType service, String hostname, int port) throws IOException {
161     hBaseClusterManager.kill(service, hostname, port);
162   }
163 
164   @Override
165   public void resume(ServiceType service, String hostname, int port) throws IOException {
166     hBaseClusterManager.kill(service, hostname, port);
167   }
168 
169 
170   // Convenience method to execute command against role on hostname. Only graceful commands are
171   // supported since cluster management APIs don't tend to let you SIGKILL things.
172   private void performClusterManagerCommand(ServiceType role, String hostname, RoleCommand command)
173       throws IOException {
174     LOG.info("Performing " + command + " command against " + role + " on " + hostname + "...");
175     String serviceName = getServiceName(roleServiceType.get(role));
176     String hostId = getHostId(hostname);
177     String roleName = getRoleName(serviceName, role.toString(), hostId);
178     doRoleCommand(serviceName, roleName, command);
179   }
180 
181   // Performing a command (e.g. starting or stopping a role) requires a POST instead of a GET.
182   private void doRoleCommand(String serviceName, String roleName, RoleCommand roleCommand) {
183     URI uri = UriBuilder.fromUri(serverHostname)
184         .path("api")
185         .path(API_VERSION)
186         .path("clusters")
187         .path(clusterName)
188         .path("services")
189         .path(serviceName)
190         .path("roleCommands")
191         .path(roleCommand.toString())
192         .build();
193     String body = "{ \"items\": [ \"" + roleName + "\" ] }";
194     LOG.info("Executing POST against " + uri + " with body " + body + "...");
195     ClientResponse response = client.resource(uri)
196         .type(MediaType.APPLICATION_JSON)
197         .post(ClientResponse.class, body);
198 
199     int statusCode = response.getStatus();
200     if (statusCode != Response.Status.OK.getStatusCode()) {
201       throw new HTTPException(statusCode);
202     }
203   }
204 
205   // Possible healthSummary values include "GOOD" and "BAD."
206   private String getHealthSummary(String serviceName, String roleType, String hostId)
207       throws IOException {
208     return getRolePropertyValue(serviceName, roleType, hostId, "healthSummary");
209   }
210 
211   // This API uses a hostId to execute host-specific commands; get one from a hostname.
212   private String getHostId(String hostname) throws IOException {
213     String hostId = null;
214 
215     URI uri = UriBuilder.fromUri(serverHostname)
216         .path("api")
217         .path(API_VERSION)
218         .path("hosts")
219         .build();
220     JsonNode hosts = getJsonNodeFromURIGet(uri);
221     if (hosts != null) {
222       // Iterate through the list of hosts, stopping once you've reached the requested hostname.
223       for (JsonNode host : hosts) {
224         if (host.get("hostname").getTextValue().equals(hostname)) {
225           hostId = host.get("hostId").getTextValue();
226           break;
227         }
228       }
229     } else {
230       hostId = null;
231     }
232 
233     return hostId;
234   }
235 
236   // Execute GET against URI, returning a JsonNode object to be traversed.
237   private JsonNode getJsonNodeFromURIGet(URI uri) throws IOException {
238     LOG.info("Executing GET against " + uri + "...");
239     ClientResponse response = client.resource(uri)
240         .accept(MediaType.APPLICATION_JSON_TYPE)
241         .get(ClientResponse.class);
242 
243     int statusCode = response.getStatus();
244     if (statusCode != Response.Status.OK.getStatusCode()) {
245       throw new HTTPException(statusCode);
246     }
247     // This API folds information as the value to an "items" attribute.
248     return new ObjectMapper().readTree(response.getEntity(String.class)).get("items");
249   }
250 
251   // This API assigns a unique role name to each host's instance of a role.
252   private String getRoleName(String serviceName, String roleType, String hostId)
253       throws IOException {
254     return getRolePropertyValue(serviceName, roleType, hostId, "name");
255   }
256 
257   // Get the value of a  property from a role on a particular host.
258   private String getRolePropertyValue(String serviceName, String roleType, String hostId,
259       String property) throws IOException {
260     String roleValue = null;
261     URI uri = UriBuilder.fromUri(serverHostname)
262         .path("api")
263         .path(API_VERSION)
264         .path("clusters")
265         .path(clusterName)
266         .path("services")
267         .path(serviceName)
268         .path("roles")
269         .build();
270     JsonNode roles = getJsonNodeFromURIGet(uri);
271     if (roles != null) {
272       // Iterate through the list of roles, stopping once the requested one is found.
273       for (JsonNode role : roles) {
274         if (role.get("hostRef").get("hostId").getTextValue().equals(hostId) &&
275             role.get("type")
276                 .getTextValue()
277                 .toLowerCase()
278                 .equals(roleType.toLowerCase())) {
279           roleValue = role.get(property).getTextValue();
280           break;
281         }
282       }
283     }
284 
285     return roleValue;
286   }
287 
288   // Possible roleState values include "STARTED" and "STOPPED."
289   private String getRoleState(String serviceName, String roleType, String hostId)
290       throws IOException {
291     return getRolePropertyValue(serviceName, roleType, hostId, "roleState");
292   }
293 
294   // Convert a service (e.g. "HBASE," "HDFS") into a service name (e.g. "HBASE-1," "HDFS-1").
295   private String getServiceName(Service service) throws IOException {
296     String serviceName = null;
297     URI uri = UriBuilder.fromUri(serverHostname)
298         .path("api")
299         .path(API_VERSION)
300         .path("clusters")
301         .path(clusterName)
302         .path("services")
303         .build();
304     JsonNode services = getJsonNodeFromURIGet(uri);
305     if (services != null) {
306       // Iterate through the list of services, stopping once the requested one is found.
307       for (JsonNode serviceEntry : services) {
308         if (serviceEntry.get("type").getTextValue().equals(service.toString())) {
309           serviceName = serviceEntry.get("name").getTextValue();
310           break;
311         }
312       }
313     }
314 
315     return serviceName;
316   }
317 
318   /*
319    * Some enums to guard against bad calls.
320    */
321 
322   // The RoleCommand enum is used by the doRoleCommand method to guard against non-existent methods
323   // being invoked on a given role.
324   private enum RoleCommand {
325     START, STOP, RESTART;
326 
327     // APIs tend to take commands in lowercase, so convert them to save the trouble later.
328     @Override
329     public String toString() {
330       return name().toLowerCase();
331     }
332   }
333 
334   // ClusterManager methods take a "ServiceType" object (e.g. "HBASE_MASTER," "HADOOP_NAMENODE").
335   // These "service types," which cluster managers call "roles" or "components," need to be mapped
336   // to their corresponding service (e.g. "HBase," "HDFS") in order to be controlled.
337   private static Map<ServiceType, Service> roleServiceType = new HashMap<ServiceType, Service>();
338   static {
339     roleServiceType.put(ServiceType.HADOOP_NAMENODE, Service.HDFS);
340     roleServiceType.put(ServiceType.HADOOP_DATANODE, Service.HDFS);
341     roleServiceType.put(ServiceType.HADOOP_JOBTRACKER, Service.MAPREDUCE);
342     roleServiceType.put(ServiceType.HADOOP_TASKTRACKER, Service.MAPREDUCE);
343     roleServiceType.put(ServiceType.HBASE_MASTER, Service.HBASE);
344     roleServiceType.put(ServiceType.HBASE_REGIONSERVER, Service.HBASE);
345   }
346 
347   private enum Service {
348     HBASE, HDFS, MAPREDUCE
349   }
350 }