View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.protobuf;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.Cell;
28  import org.apache.hadoop.hbase.CellScanner;
29  import org.apache.hadoop.hbase.DoNotRetryIOException;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.ServerName;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.ipc.ServerRpcController;
34  import org.apache.hadoop.hbase.protobuf.generated.AccessControlProtos.GetUserPermissionsResponse;
35  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CloseRegionResponse;
36  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetOnlineRegionResponse;
37  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
38  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
39  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterResponse;
40  import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
41  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
42  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
43  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionAction;
44  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultOrException;
46  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
47  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiResponse;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
50  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.EnableCatalogJanitorResponse;
51  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanResponse;
52  import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
53  import org.apache.hadoop.hbase.regionserver.RegionOpeningState;
54  import org.apache.hadoop.hbase.security.access.UserPermission;
55  import org.apache.hadoop.hbase.util.Pair;
56  import org.apache.hadoop.util.StringUtils;
57  
58  import com.google.protobuf.ByteString;
59  import com.google.protobuf.RpcController;
60  
61  /**
62   * Helper utility to build protocol buffer responses,
63   * or retrieve data from protocol buffer responses.
64   */
65  @InterfaceAudience.Private
66  public final class ResponseConverter {
67    public static final Log LOG = LogFactory.getLog(ResponseConverter.class);
68  
69    private ResponseConverter() {
70    }
71  
72  // Start utilities for Client
73  
74    /**
75     * Get the results from a protocol buffer MultiResponse
76     *
77     * @param request the protocol buffer MultiResponse to convert
78     * @param cells Cells to go with the passed in <code>proto</code>.  Can be null.
79     * @return the results that were in the MultiResponse (a Result or an Exception).
80     * @throws IOException
81     */
82    public static org.apache.hadoop.hbase.client.MultiResponse getResults(final MultiRequest request,
83        final MultiResponse response, final CellScanner cells)
84    throws IOException {
85      int requestRegionActionCount = request.getRegionActionCount();
86      int responseRegionActionResultCount = response.getRegionActionResultCount();
87      if (requestRegionActionCount != responseRegionActionResultCount) {
88        throw new IllegalStateException("Request mutation count=" + responseRegionActionResultCount +
89            " does not match response mutation result count=" + responseRegionActionResultCount);
90      }
91  
92      org.apache.hadoop.hbase.client.MultiResponse results =
93        new org.apache.hadoop.hbase.client.MultiResponse();
94  
95      for (int i = 0; i < responseRegionActionResultCount; i++) {
96        RegionAction actions = request.getRegionAction(i);
97        RegionActionResult actionResult = response.getRegionActionResult(i);
98        HBaseProtos.RegionSpecifier rs = actions.getRegion();
99        if (rs.hasType() &&
100           (rs.getType() != HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME)){
101         throw new IllegalArgumentException(
102             "We support only encoded types for protobuf multi response.");
103       }
104       byte[] regionName = rs.getValue().toByteArray();
105 
106       if (actionResult.hasException()){
107         Throwable regionException =  ProtobufUtil.toException(actionResult.getException());
108         results.addException(regionName, regionException);
109         continue;
110       }
111 
112       if (actions.getActionCount() != actionResult.getResultOrExceptionCount()) {
113         throw new IllegalStateException("actions.getActionCount=" + actions.getActionCount() +
114             ", actionResult.getResultOrExceptionCount=" +
115             actionResult.getResultOrExceptionCount() + " for region " + actions.getRegion());
116       }
117 
118       for (ResultOrException roe : actionResult.getResultOrExceptionList()) {
119         Object responseValue;
120         if (roe.hasException()) {
121           responseValue = ProtobufUtil.toException(roe.getException());
122         } else if (roe.hasResult()) {
123           responseValue = ProtobufUtil.toResult(roe.getResult(), cells);
124           // add the load stats, if we got any
125           if (roe.hasLoadStats()) {
126             ((Result) responseValue).addResults(roe.getLoadStats());
127           }
128         } else if (roe.hasServiceResult()) {
129           responseValue = roe.getServiceResult();
130         } else {
131           // no result & no exception. Unexpected.
132           throw new IllegalStateException("No result & no exception roe=" + roe +
133               " for region " + actions.getRegion());
134         }
135         results.add(regionName, roe.getIndex(), responseValue);
136       }
137     }
138 
139     return results;
140   }
141 
142   /**
143    * Wrap a throwable to an action result.
144    *
145    * @param t
146    * @return an action result builder
147    */
148   public static ResultOrException.Builder buildActionResult(final Throwable t) {
149     ResultOrException.Builder builder = ResultOrException.newBuilder();
150     if (t != null) builder.setException(buildException(t));
151     return builder;
152   }
153 
154   /**
155    * Wrap a throwable to an action result.
156    *
157    * @param r
158    * @param stats
159    * @return an action result builder
160    */
161   public static ResultOrException.Builder buildActionResult(final ClientProtos.Result r,
162       ClientProtos.RegionLoadStats stats) {
163     ResultOrException.Builder builder = ResultOrException.newBuilder();
164     if (r != null) builder.setResult(r);
165     if (stats != null) builder.setLoadStats(stats);
166     return builder;
167   }
168 
169   /**
170    * @param t
171    * @return NameValuePair of the exception name to stringified version os exception.
172    */
173   public static NameBytesPair buildException(final Throwable t) {
174     NameBytesPair.Builder parameterBuilder = NameBytesPair.newBuilder();
175     parameterBuilder.setName(t.getClass().getName());
176     parameterBuilder.setValue(
177       ByteString.copyFromUtf8(StringUtils.stringifyException(t)));
178     return parameterBuilder.build();
179   }
180 
181   /**
182    * Converts the permissions list into a protocol buffer GetUserPermissionsResponse
183    */
184   public static GetUserPermissionsResponse buildGetUserPermissionsResponse(
185       final List<UserPermission> permissions) {
186     GetUserPermissionsResponse.Builder builder = GetUserPermissionsResponse.newBuilder();
187     for (UserPermission perm : permissions) {
188       builder.addUserPermission(ProtobufUtil.toUserPermission(perm));
189     }
190     return builder.build();
191   }
192 
193 // End utilities for Client
194 // Start utilities for Admin
195 
196   /**
197    * Get the list of regions to flush from a RollLogWriterResponse
198    *
199    * @param proto the RollLogWriterResponse
200    * @return the the list of regions to flush
201    */
202   public static byte[][] getRegions(final RollWALWriterResponse proto) {
203     if (proto == null || proto.getRegionToFlushCount() == 0) return null;
204     List<byte[]> regions = new ArrayList<byte[]>();
205     for (ByteString region: proto.getRegionToFlushList()) {
206       regions.add(region.toByteArray());
207     }
208     return (byte[][])regions.toArray();
209   }
210 
211   /**
212    * Get the list of region info from a GetOnlineRegionResponse
213    *
214    * @param proto the GetOnlineRegionResponse
215    * @return the list of region info
216    */
217   public static List<HRegionInfo> getRegionInfos(final GetOnlineRegionResponse proto) {
218     if (proto == null || proto.getRegionInfoCount() == 0) return null;
219     return ProtobufUtil.getRegionInfos(proto);
220   }
221 
222   /**
223    * Get the region opening state from a OpenRegionResponse
224    *
225    * @param proto the OpenRegionResponse
226    * @return the region opening state
227    */
228   public static RegionOpeningState getRegionOpeningState
229       (final OpenRegionResponse proto) {
230     if (proto == null || proto.getOpeningStateCount() != 1) return null;
231     return RegionOpeningState.valueOf(
232       proto.getOpeningState(0).name());
233   }
234 
235   /**
236    * Get a list of region opening state from a OpenRegionResponse
237    * 
238    * @param proto the OpenRegionResponse
239    * @return the list of region opening state
240    */
241   public static List<RegionOpeningState> getRegionOpeningStateList(
242       final OpenRegionResponse proto) {
243     if (proto == null) return null;
244     List<RegionOpeningState> regionOpeningStates = new ArrayList<RegionOpeningState>();
245     for (int i = 0; i < proto.getOpeningStateCount(); i++) {
246       regionOpeningStates.add(RegionOpeningState.valueOf(
247           proto.getOpeningState(i).name()));
248     }
249     return regionOpeningStates;
250   }
251 
252   /**
253    * Check if the region is closed from a CloseRegionResponse
254    *
255    * @param proto the CloseRegionResponse
256    * @return the region close state
257    */
258   public static boolean isClosed
259       (final CloseRegionResponse proto) {
260     if (proto == null || !proto.hasClosed()) return false;
261     return proto.getClosed();
262   }
263 
264   /**
265    * A utility to build a GetServerInfoResponse.
266    *
267    * @param serverName
268    * @param webuiPort
269    * @return the response
270    */
271   public static GetServerInfoResponse buildGetServerInfoResponse(
272       final ServerName serverName, final int webuiPort) {
273     GetServerInfoResponse.Builder builder = GetServerInfoResponse.newBuilder();
274     ServerInfo.Builder serverInfoBuilder = ServerInfo.newBuilder();
275     serverInfoBuilder.setServerName(ProtobufUtil.toServerName(serverName));
276     if (webuiPort >= 0) {
277       serverInfoBuilder.setWebuiPort(webuiPort);
278     }
279     builder.setServerInfo(serverInfoBuilder.build());
280     return builder.build();
281   }
282 
283   /**
284    * A utility to build a GetOnlineRegionResponse.
285    *
286    * @param regions
287    * @return the response
288    */
289   public static GetOnlineRegionResponse buildGetOnlineRegionResponse(
290       final List<HRegionInfo> regions) {
291     GetOnlineRegionResponse.Builder builder = GetOnlineRegionResponse.newBuilder();
292     for (HRegionInfo region: regions) {
293       builder.addRegionInfo(HRegionInfo.convert(region));
294     }
295     return builder.build();
296   }
297 
298   /**
299    * Creates a response for the catalog scan request
300    * @return A RunCatalogScanResponse
301    */
302   public static RunCatalogScanResponse buildRunCatalogScanResponse(int numCleaned) {
303     return RunCatalogScanResponse.newBuilder().setScanResult(numCleaned).build();
304   }
305 
306   /**
307    * Creates a response for the catalog scan request
308    * @return A EnableCatalogJanitorResponse
309    */
310   public static EnableCatalogJanitorResponse buildEnableCatalogJanitorResponse(boolean prevValue) {
311     return EnableCatalogJanitorResponse.newBuilder().setPrevValue(prevValue).build();
312   }
313 
314 // End utilities for Admin
315 
316   /**
317    * Creates a response for the last flushed sequence Id request
318    * @return A GetLastFlushedSequenceIdResponse
319    */
320   public static GetLastFlushedSequenceIdResponse buildGetLastFlushedSequenceIdResponse(
321       long seqId) {
322     return GetLastFlushedSequenceIdResponse.newBuilder().setLastFlushedSequenceId(seqId).build();
323   }
324 
325   /**
326    * Stores an exception encountered during RPC invocation so it can be passed back
327    * through to the client.
328    * @param controller the controller instance provided by the client when calling the service
329    * @param ioe the exception encountered
330    */
331   public static void setControllerException(RpcController controller, IOException ioe) {
332     if (controller != null) {
333       if (controller instanceof ServerRpcController) {
334         ((ServerRpcController)controller).setFailedOn(ioe);
335       } else {
336         controller.setFailed(StringUtils.stringifyException(ioe));
337       }
338     }
339   }
340 
341   /**
342    * Create Results from the cells using the cells meta data. 
343    * @param cellScanner
344    * @param response
345    * @return results
346    */
347   public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
348       throws IOException {
349     if (response == null) return null;
350     // If cellscanner, then the number of Results to return is the count of elements in the
351     // cellsPerResult list.  Otherwise, it is how many results are embedded inside the response.
352     int noOfResults = cellScanner != null?
353       response.getCellsPerResultCount(): response.getResultsCount();
354     Result[] results = new Result[noOfResults];
355     for (int i = 0; i < noOfResults; i++) {
356       if (cellScanner != null) {
357         // Cells are out in cellblocks.  Group them up again as Results.  How many to read at a
358         // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
359         int noOfCells = response.getCellsPerResult(i);
360         List<Cell> cells = new ArrayList<Cell>(noOfCells);
361         for (int j = 0; j < noOfCells; j++) {
362           try {
363             if (cellScanner.advance() == false) {
364               // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
365               // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
366               // same scanner will result in OutOfOrderScannerNextException
367               String msg = "Results sent from server=" + noOfResults + ". But only got " + i
368                 + " results completely at client. Resetting the scanner to scan again.";
369               LOG.error(msg);
370               throw new DoNotRetryIOException(msg);
371             }
372           } catch (IOException ioe) {
373             // We are getting IOE while retrieving the cells for Results.
374             // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
375             // same scanner will result in OutOfOrderScannerNextException
376             LOG.error("Exception while reading cells from result."
377               + "Resetting the scanner to scan again.", ioe);
378             throw new DoNotRetryIOException("Resetting the scanner.", ioe);
379           }
380           cells.add(cellScanner.current());
381         }
382         results[i] = Result.create(cells);
383       } else {
384         // Result is pure pb.
385         results[i] = ProtobufUtil.toResult(response.getResults(i));
386       }
387     }
388     return results;
389   }
390 }