1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import static org.junit.Assert.*;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Set;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.hbase.*;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.client.Durability;
38 import org.apache.hadoop.hbase.testclassification.SmallTests;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.junit.Rule;
41 import org.junit.Test;
42 import org.junit.experimental.categories.Category;
43 import org.junit.rules.TestName;
44
45 @Category(SmallTests.class)
46 public class TestColumnSeeking {
47 @Rule public TestName name = new TestName();
48
49 private final static HBaseTestingUtility TEST_UTIL = HBaseTestingUtility.createLocalHTU();
50
51 static final Log LOG = LogFactory.getLog(TestColumnSeeking.class);
52
53 @SuppressWarnings("unchecked")
54 @Test
55 public void testDuplicateVersions() throws IOException {
56 String family = "Family";
57 byte[] familyBytes = Bytes.toBytes("Family");
58 TableName table = TableName.valueOf(name.getMethodName());
59
60 HColumnDescriptor hcd =
61 new HColumnDescriptor(familyBytes).setMaxVersions(1000);
62 hcd.setMaxVersions(3);
63 HTableDescriptor htd = new HTableDescriptor(table);
64 htd.addFamily(hcd);
65 HRegionInfo info = new HRegionInfo(table, null, null, false);
66
67 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
68 try {
69 List<String> rows = generateRandomWords(10, "row");
70 List<String> allColumns = generateRandomWords(10, "column");
71 List<String> values = generateRandomWords(100, "value");
72
73 long maxTimestamp = 2;
74 double selectPercent = 0.5;
75 int numberOfTests = 5;
76 double flushPercentage = 0.2;
77 double minorPercentage = 0.2;
78 double majorPercentage = 0.2;
79 double putPercentage = 0.2;
80
81 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
82
83 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
84 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
85
86 for (int i = 0; i < numberOfTests; i++) {
87 kvMaps[i] = new HashMap<String, KeyValue>();
88 columnLists[i] = new ArrayList<String>();
89 for (String column : allColumns) {
90 if (Math.random() < selectPercent) {
91 columnLists[i].add(column);
92 }
93 }
94 }
95
96 for (String value : values) {
97 for (String row : rows) {
98 Put p = new Put(Bytes.toBytes(row));
99 p.setDurability(Durability.SKIP_WAL);
100 for (String column : allColumns) {
101 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
102 KeyValue kv =
103 KeyValueTestUtil.create(row, family, column, timestamp, value);
104 if (Math.random() < putPercentage) {
105 p.add(kv);
106 allKVMap.put(kv.getKeyString(), kv);
107 for (int i = 0; i < numberOfTests; i++) {
108 if (columnLists[i].contains(column)) {
109 kvMaps[i].put(kv.getKeyString(), kv);
110 }
111 }
112 }
113 }
114 }
115 region.put(p);
116 if (Math.random() < flushPercentage) {
117 LOG.info("Flushing... ");
118 region.flushcache();
119 }
120
121 if (Math.random() < minorPercentage) {
122 LOG.info("Minor compacting... ");
123 region.compactStores(false);
124 }
125
126 if (Math.random() < majorPercentage) {
127 LOG.info("Major compacting... ");
128 region.compactStores(true);
129 }
130 }
131 }
132
133 for (int i = 0; i < numberOfTests + 1; i++) {
134 Collection<KeyValue> kvSet;
135 Scan scan = new Scan();
136 scan.setMaxVersions();
137 if (i < numberOfTests) {
138 if (columnLists[i].size() == 0) continue;
139 kvSet = kvMaps[i].values();
140 for (String column : columnLists[i]) {
141 scan.addColumn(familyBytes, Bytes.toBytes(column));
142 }
143 LOG.info("ExplicitColumns scanner");
144 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
145 + kvSet.size());
146 } else {
147 kvSet = allKVMap.values();
148 LOG.info("Wildcard scanner");
149 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
150
151 }
152 InternalScanner scanner = region.getScanner(scan);
153 List<Cell> results = new ArrayList<Cell>();
154 while (scanner.next(results))
155 ;
156 assertEquals(kvSet.size(), results.size());
157 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
158 }
159 } finally {
160 HRegion.closeHRegion(region);
161 }
162
163 HRegion.closeHRegion(region);
164 }
165
166 @SuppressWarnings("unchecked")
167 @Test
168 public void testReseeking() throws IOException {
169 String family = "Family";
170 byte[] familyBytes = Bytes.toBytes("Family");
171 TableName table = TableName.valueOf(name.getMethodName());
172
173 HTableDescriptor htd = new HTableDescriptor(table);
174 HColumnDescriptor hcd = new HColumnDescriptor(family);
175 hcd.setMaxVersions(3);
176 htd.addFamily(hcd);
177
178 HRegionInfo info = new HRegionInfo(table, null, null, false);
179 HRegion region = TEST_UTIL.createLocalHRegion(info, htd);
180
181 List<String> rows = generateRandomWords(10, "row");
182 List<String> allColumns = generateRandomWords(100, "column");
183
184 long maxTimestamp = 2;
185 double selectPercent = 0.5;
186 int numberOfTests = 5;
187 double flushPercentage = 0.2;
188 double minorPercentage = 0.2;
189 double majorPercentage = 0.2;
190 double putPercentage = 0.2;
191
192 HashMap<String, KeyValue> allKVMap = new HashMap<String, KeyValue>();
193
194 HashMap<String, KeyValue>[] kvMaps = new HashMap[numberOfTests];
195 ArrayList<String>[] columnLists = new ArrayList[numberOfTests];
196 String valueString = "Value";
197
198 for (int i = 0; i < numberOfTests; i++) {
199 kvMaps[i] = new HashMap<String, KeyValue>();
200 columnLists[i] = new ArrayList<String>();
201 for (String column : allColumns) {
202 if (Math.random() < selectPercent) {
203 columnLists[i].add(column);
204 }
205 }
206 }
207
208 for (String row : rows) {
209 Put p = new Put(Bytes.toBytes(row));
210 p.setDurability(Durability.SKIP_WAL);
211 for (String column : allColumns) {
212 for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
213 KeyValue kv =
214 KeyValueTestUtil.create(row, family, column, timestamp,
215 valueString);
216 if (Math.random() < putPercentage) {
217 p.add(kv);
218 allKVMap.put(kv.getKeyString(), kv);
219 for (int i = 0; i < numberOfTests; i++) {
220 if (columnLists[i].contains(column)) {
221 kvMaps[i].put(kv.getKeyString(), kv);
222 }
223 }
224 }
225
226 }
227 }
228 region.put(p);
229 if (Math.random() < flushPercentage) {
230 LOG.info("Flushing... ");
231 region.flushcache();
232 }
233
234 if (Math.random() < minorPercentage) {
235 LOG.info("Minor compacting... ");
236 region.compactStores(false);
237 }
238
239 if (Math.random() < majorPercentage) {
240 LOG.info("Major compacting... ");
241 region.compactStores(true);
242 }
243 }
244
245 for (int i = 0; i < numberOfTests + 1; i++) {
246 Collection<KeyValue> kvSet;
247 Scan scan = new Scan();
248 scan.setMaxVersions();
249 if (i < numberOfTests) {
250 if (columnLists[i].size() == 0) continue;
251 kvSet = kvMaps[i].values();
252 for (String column : columnLists[i]) {
253 scan.addColumn(familyBytes, Bytes.toBytes(column));
254 }
255 LOG.info("ExplicitColumns scanner");
256 LOG.info("Columns: " + columnLists[i].size() + " Keys: "
257 + kvSet.size());
258 } else {
259 kvSet = allKVMap.values();
260 LOG.info("Wildcard scanner");
261 LOG.info("Columns: " + allColumns.size() + " Keys: " + kvSet.size());
262
263 }
264 InternalScanner scanner = region.getScanner(scan);
265 List<Cell> results = new ArrayList<Cell>();
266 while (scanner.next(results))
267 ;
268 assertEquals(kvSet.size(), results.size());
269 assertTrue(KeyValueTestUtil.containsIgnoreMvccVersion(results, kvSet));
270 }
271
272 HRegion.closeHRegion(region);
273 }
274
275 List<String> generateRandomWords(int numberOfWords, String suffix) {
276 Set<String> wordSet = new HashSet<String>();
277 for (int i = 0; i < numberOfWords; i++) {
278 int lengthOfWords = (int) (Math.random() * 5) + 1;
279 char[] wordChar = new char[lengthOfWords];
280 for (int j = 0; j < wordChar.length; j++) {
281 wordChar[j] = (char) (Math.random() * 26 + 97);
282 }
283 String word;
284 if (suffix == null) {
285 word = new String(wordChar);
286 } else {
287 word = new String(wordChar) + suffix;
288 }
289 wordSet.add(word);
290 }
291 List<String> wordList = new ArrayList<String>(wordSet);
292 return wordList;
293 }
294
295 }
296