View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.replication;
20  
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.TreeMap;
25  
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.hbase.HRegionInfo;
28  import org.apache.hadoop.hbase.HTableDescriptor;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
33  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
34  import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
35  import org.apache.hadoop.hbase.testclassification.SmallTests;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.Assert;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  
41  import com.google.common.collect.Lists;
42  
43  import static org.junit.Assert.*;
44  import static org.mockito.Mockito.*;
45  
46  @Category(SmallTests.class)
47  public class TestReplicationWALEntryFilters {
48  
49    static byte[] a = new byte[] {'a'};
50    static byte[] b = new byte[] {'b'};
51    static byte[] c = new byte[] {'c'};
52    static byte[] d = new byte[] {'d'};
53  
54    @Test
55    public void testSystemTableWALEntryFilter() {
56      SystemTableWALEntryFilter filter = new SystemTableWALEntryFilter();
57  
58      // meta
59      HLogKey key1 = new HLogKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(),
60        HTableDescriptor.META_TABLEDESC.getTableName(), 0, 0, null);
61      HLog.Entry metaEntry = new Entry(key1, null);
62  
63      assertNull(filter.filter(metaEntry));
64  
65      // ns table
66      HLogKey key2 = new HLogKey(new byte[] {}, HTableDescriptor.NAMESPACE_TABLEDESC.getTableName()
67        , 0, 0, null);
68      HLog.Entry nsEntry = new Entry(key2, null);
69      assertNull(filter.filter(nsEntry));
70  
71      // user table
72  
73      HLogKey key3 = new HLogKey(new byte[] {}, TableName.valueOf("foo"), 0, 0, null);
74      HLog.Entry userEntry = new Entry(key3, null);
75  
76      assertEquals(userEntry, filter.filter(userEntry));
77    }
78  
79    @Test
80    public void testScopeWALEntryFilter() {
81      ScopeWALEntryFilter filter = new ScopeWALEntryFilter();
82  
83      HLog.Entry userEntry = createEntry(a, b);
84      HLog.Entry userEntryA = createEntry(a);
85      HLog.Entry userEntryB = createEntry(b);
86      HLog.Entry userEntryEmpty = createEntry();
87  
88      // no scopes
89      assertEquals(null, filter.filter(userEntry));
90  
91      // empty scopes
92      TreeMap<byte[], Integer> scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
93      userEntry = createEntry(a, b);
94      userEntry.getKey().setScopes(scopes);
95      assertEquals(null, filter.filter(userEntry));
96  
97      // different scope
98      scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
99      scopes.put(c, HConstants.REPLICATION_SCOPE_GLOBAL);
100     userEntry = createEntry(a, b);
101     userEntry.getKey().setScopes(scopes);
102     // all kvs should be filtered
103     assertEquals(userEntryEmpty, filter.filter(userEntry));
104 
105     // local scope
106     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
107     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
108     userEntry = createEntry(a, b);
109     userEntry.getKey().setScopes(scopes);
110     assertEquals(userEntryEmpty, filter.filter(userEntry));
111     scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
112     assertEquals(userEntryEmpty, filter.filter(userEntry));
113 
114     // only scope a
115     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
116     scopes.put(a, HConstants.REPLICATION_SCOPE_GLOBAL);
117     userEntry = createEntry(a, b);
118     userEntry.getKey().setScopes(scopes);
119     assertEquals(userEntryA, filter.filter(userEntry));
120     scopes.put(b, HConstants.REPLICATION_SCOPE_LOCAL);
121     assertEquals(userEntryA, filter.filter(userEntry));
122 
123     // only scope b
124     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
125     scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
126     userEntry = createEntry(a, b);
127     userEntry.getKey().setScopes(scopes);
128     assertEquals(userEntryB, filter.filter(userEntry));
129     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
130     assertEquals(userEntryB, filter.filter(userEntry));
131 
132     // scope a and b
133     scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
134     scopes.put(b, HConstants.REPLICATION_SCOPE_GLOBAL);
135     userEntry = createEntry(a, b);
136     userEntry.getKey().setScopes(scopes);
137     assertEquals(userEntryB, filter.filter(userEntry));
138     scopes.put(a, HConstants.REPLICATION_SCOPE_LOCAL);
139     assertEquals(userEntryB, filter.filter(userEntry));
140   }
141 
142   WALEntryFilter nullFilter = new WALEntryFilter() {
143     @Override
144     public Entry filter(Entry entry) {
145       return null;
146     }
147   };
148 
149   WALEntryFilter passFilter = new WALEntryFilter() {
150     @Override
151     public Entry filter(Entry entry) {
152       return entry;
153     }
154   };
155 
156   @Test
157   public void testChainWALEntryFilter() {
158     HLog.Entry userEntry = createEntry(a, b, c);
159 
160     ChainWALEntryFilter filter = new ChainWALEntryFilter(passFilter);
161     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
162 
163     filter = new ChainWALEntryFilter(passFilter, passFilter);
164     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
165 
166     filter = new ChainWALEntryFilter(passFilter, passFilter, passFilter);
167     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
168 
169     filter = new ChainWALEntryFilter(nullFilter);
170     assertEquals(null, filter.filter(userEntry));
171 
172     filter = new ChainWALEntryFilter(nullFilter, passFilter);
173     assertEquals(null, filter.filter(userEntry));
174 
175     filter = new ChainWALEntryFilter(passFilter, nullFilter);
176     assertEquals(null, filter.filter(userEntry));
177 
178     filter = new ChainWALEntryFilter(nullFilter, passFilter, nullFilter);
179     assertEquals(null, filter.filter(userEntry));
180 
181     filter = new ChainWALEntryFilter(nullFilter, nullFilter);
182     assertEquals(null, filter.filter(userEntry));
183 
184     // flatten
185     filter =
186         new ChainWALEntryFilter(
187           new ChainWALEntryFilter(passFilter,
188             new ChainWALEntryFilter(passFilter, passFilter),
189           new ChainWALEntryFilter(passFilter),
190           new ChainWALEntryFilter(passFilter)),
191           new ChainWALEntryFilter(passFilter));
192     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
193 
194 
195     filter =
196         new ChainWALEntryFilter(
197           new ChainWALEntryFilter(passFilter,
198             new ChainWALEntryFilter(passFilter,
199               new ChainWALEntryFilter(nullFilter))),
200           new ChainWALEntryFilter(passFilter));
201     assertEquals(null, filter.filter(userEntry));
202   }
203 
204   @Test
205   public void testTableCfWALEntryFilter() {
206     ReplicationPeer peer = mock(ReplicationPeer.class);
207 
208     when(peer.getTableCFs()).thenReturn(null);
209     HLog.Entry userEntry = createEntry(a, b, c);
210     TableCfWALEntryFilter filter = new TableCfWALEntryFilter(peer);
211     assertEquals(createEntry(a,b,c), filter.filter(userEntry));
212 
213     // empty map
214     userEntry = createEntry(a, b, c);
215     Map<String, List<String>> tableCfs = new HashMap<String, List<String>>();
216     when(peer.getTableCFs()).thenReturn(tableCfs);
217     filter = new TableCfWALEntryFilter(peer);
218     assertEquals(null, filter.filter(userEntry));
219 
220     // table bar
221     userEntry = createEntry(a, b, c);
222     tableCfs = new HashMap<String, List<String>>();
223     tableCfs.put("bar", null);
224     when(peer.getTableCFs()).thenReturn(tableCfs);
225     filter = new TableCfWALEntryFilter(peer);
226     assertEquals(null, filter.filter(userEntry));
227 
228     // table foo:a
229     userEntry = createEntry(a, b, c);
230     tableCfs = new HashMap<String, List<String>>();
231     tableCfs.put("foo", Lists.newArrayList("a"));
232     when(peer.getTableCFs()).thenReturn(tableCfs);
233     filter = new TableCfWALEntryFilter(peer);
234     assertEquals(createEntry(a), filter.filter(userEntry));
235 
236     // table foo:a,c
237     userEntry = createEntry(a, b, c, d);
238     tableCfs = new HashMap<String, List<String>>();
239     tableCfs.put("foo", Lists.newArrayList("a", "c"));
240     when(peer.getTableCFs()).thenReturn(tableCfs);
241     filter = new TableCfWALEntryFilter(peer);
242     assertEquals(createEntry(a,c), filter.filter(userEntry));
243   }
244 
245   private HLog.Entry createEntry(byte[]... kvs) {
246     HLogKey key1 = new HLogKey(new byte[] {}, TableName.valueOf("foo"), 0, 0, null);
247     WALEdit edit1 = new WALEdit();
248 
249     for (byte[] kv : kvs) {
250       edit1.add(new KeyValue(kv, kv, kv));
251     }
252     return new HLog.Entry(key1, edit1);
253   }
254 
255 
256   private void assertEquals(HLog.Entry e1, HLog.Entry e2) {
257     Assert.assertEquals(e1 == null, e2 == null);
258     if (e1 == null) {
259       return;
260     }
261 
262     // do not compare HLogKeys
263 
264     // compare kvs
265     Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
266     if (e1.getEdit() == null) {
267       return;
268     }
269     List<KeyValue> kvs1 = e1.getEdit().getKeyValues();
270     List<KeyValue> kvs2 = e2.getEdit().getKeyValues();
271     Assert.assertEquals(kvs1.size(), kvs2.size());
272     for (int i = 0; i < kvs1.size(); i++) {
273       KeyValue.COMPARATOR.compare(kvs1.get(i), kvs2.get(i));
274     }
275   }
276 
277 
278 }