View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  
23  import java.util.ArrayList;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Map.Entry;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.testclassification.LargeTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.AfterClass;
38  import org.junit.BeforeClass;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  import com.google.common.collect.Maps;
43  
44  @Category(LargeTests.class)
45  public class TestSizeFailures {
46    static final Log LOG = LogFactory.getLog(TestSizeFailures.class);
47    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
48    private static byte[] FAMILY = Bytes.toBytes("testFamily");
49    protected static int SLAVES = 1;
50    private static HTable table;
51    private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10;
52  
53    @BeforeClass
54    public static void setUpBeforeClass() throws Exception {
55      // Uncomment the following lines if more verbosity is needed for
56      // debugging (see HBASE-12285 for details).
57      // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
58      // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
59      // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
60      Configuration conf = TEST_UTIL.getConfiguration();
61      conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server
62      TEST_UTIL.startMiniCluster(SLAVES);
63  
64      // Write a bunch of data
65      final TableName TABLENAME = TableName.valueOf("testSizeFailures");
66      List<byte[]> qualifiers = new ArrayList<byte[]>();
67      for (int i = 1; i <= 10; i++) {
68        qualifiers.add(Bytes.toBytes(Integer.toString(i)));
69      }
70  
71      HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
72      HTableDescriptor desc = new HTableDescriptor(TABLENAME);
73      desc.addFamily(hcd);
74      byte[][] splits = new byte[9][2];
75      for (int i = 1; i < 10; i++) {
76        int split = 48 + i;
77        splits[i - 1][0] = (byte) (split >>> 8);
78        splits[i - 1][0] = (byte) (split);
79      }
80  
81      table = TEST_UTIL.createTable(desc, splits);
82  
83      List<Put> puts = new LinkedList<Put>();
84      for (int i = 0; i < NUM_ROWS; i++) {
85        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
86        for (int j = 0; j < NUM_COLS; j++) {
87          byte[] value = new byte[50];
88          Bytes.random(value);
89          p.add(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
90        }
91        puts.add(p);
92  
93        if (puts.size() == 1000) {
94          table.batch(puts, new Object[1000]);
95          puts.clear();
96        }
97      }
98  
99      if (puts.size() > 0) {
100       table.batch(puts, new Object[puts.size()]);
101     }
102   }
103 
104   @AfterClass
105   public static void tearDownAfterClass() throws Exception {
106     TEST_UTIL.shutdownMiniCluster();
107   }
108 
109   /**
110    * Basic client side validation of HBASE-13262
111    */
112   @Test
113   public void testScannerSeesAllRecords() throws Exception {
114    Scan s = new Scan();
115    s.addFamily(FAMILY);
116    s.setMaxResultSize(-1);
117    s.setBatch(-1);
118    s.setCaching(500);
119    Entry<Long,Long> entry = sumTable(table.getScanner(s));
120    long rowsObserved = entry.getKey();
121    long entriesObserved = entry.getValue();
122 
123    // Verify that we see 1M rows and 10M cells
124    assertEquals(NUM_ROWS, rowsObserved);
125    assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
126  }
127 
128  /**
129   * Basic client side validation of HBASE-13262
130   */
131   @Test
132   public void testSmallScannerSeesAllRecords() throws Exception {
133     Scan s = new Scan();
134     s.setSmall(true);
135     s.addFamily(FAMILY);
136     s.setMaxResultSize(-1);
137     s.setBatch(-1);
138     s.setCaching(500);
139     Entry<Long,Long> entry = sumTable(table.getScanner(s));
140     long rowsObserved = entry.getKey();
141     long entriesObserved = entry.getValue();
142 
143     // Verify that we see 1M rows and 10M cells
144     assertEquals(NUM_ROWS, rowsObserved);
145     assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
146   }
147 
148   /**
149    * Count the number of rows and the number of entries from a scanner
150    *
151    * @param scanner
152    *          The Scanner
153    * @return An entry where the first item is rows observed and the second is entries observed.
154    */
155   private Entry<Long,Long> sumTable(ResultScanner scanner) {
156     long rowsObserved = 0l;
157     long entriesObserved = 0l;
158 
159     // Read all the records in the table
160     for (Result result : scanner) {
161       rowsObserved++;
162       while (result.advance()) {
163         entriesObserved++;
164       }
165     }
166     return Maps.immutableEntry(rowsObserved,entriesObserved);
167   }
168 }