View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.commons.logging.impl.Log4JLogger;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.HBaseTestingUtility;
27  import org.apache.hadoop.hbase.HColumnDescriptor;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.HTableDescriptor;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.client.HBaseAdmin;
32  import org.apache.hadoop.hbase.client.HTable;
33  import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
34  import org.apache.hadoop.hbase.replication.regionserver.ReplicationSource;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
37  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38  import org.apache.log4j.Level;
39  import org.junit.AfterClass;
40  import org.junit.BeforeClass;
41  
42  /**
43   * This class is only a base for other integration-level replication tests.
44   * Do not add tests here.
45   * TestReplicationSmallTests is where tests that don't require bring machines up/down should go
46   * All other tests should have their own classes and extend this one
47   */
48  public class TestReplicationBase {
49  
50    {
51      ((Log4JLogger) ReplicationSource.LOG).getLogger().setLevel(Level.ALL);
52    }
53  
54    private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
55  
56    protected static Configuration conf1 = HBaseConfiguration.create();
57    protected static Configuration conf2;
58    protected static Configuration CONF_WITH_LOCALFS;
59  
60    protected static ZooKeeperWatcher zkw1;
61    protected static ZooKeeperWatcher zkw2;
62  
63    protected static ReplicationAdmin admin;
64  
65    protected static HTable htable1;
66    protected static HTable htable2;
67  
68    protected static HBaseTestingUtility utility1;
69    protected static HBaseTestingUtility utility2;
70    protected static final int NB_ROWS_IN_BATCH = 100;
71    protected static final int NB_ROWS_IN_BIG_BATCH =
72        NB_ROWS_IN_BATCH * 10;
73    protected static final long SLEEP_TIME = 500;
74    protected static final int NB_RETRIES = 10;
75  
76    protected static final byte[] tableName = Bytes.toBytes("test");
77    protected static final byte[] famName = Bytes.toBytes("f");
78    protected static final byte[] row = Bytes.toBytes("row");
79    protected static final byte[] noRepfamName = Bytes.toBytes("norep");
80  
81    /**
82     * @throws java.lang.Exception
83     */
84    @BeforeClass
85    public static void setUpBeforeClass() throws Exception {
86      conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
87      // smaller log roll size to trigger more events
88      conf1.setFloat("hbase.regionserver.logroll.multiplier", 0.0003f);
89      conf1.setInt("replication.source.size.capacity", 10240);
90      conf1.setLong("replication.source.sleepforretries", 100);
91      conf1.setInt("hbase.regionserver.maxlogs", 10);
92      conf1.setLong("hbase.master.logcleaner.ttl", 10);
93      conf1.setInt("zookeeper.recovery.retry", 1);
94      conf1.setInt("zookeeper.recovery.retry.intervalmill", 10);
95      conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
96      conf1.setBoolean("dfs.support.append", true);
97      conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
98      conf1.setInt("replication.stats.thread.period.seconds", 5);
99      conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
100 
101     utility1 = new HBaseTestingUtility(conf1);
102     utility1.startMiniZKCluster();
103     MiniZooKeeperCluster miniZK = utility1.getZkCluster();
104     // Have to reget conf1 in case zk cluster location different
105     // than default
106     conf1 = utility1.getConfiguration();  
107     zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
108     admin = new ReplicationAdmin(conf1);
109     LOG.info("Setup first Zk");
110 
111     // Base conf2 on conf1 so it gets the right zk cluster.
112     conf2 = HBaseConfiguration.create(conf1);
113     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
114     conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
115     conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
116     conf2.setBoolean("dfs.support.append", true);
117     conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false);
118 
119     utility2 = new HBaseTestingUtility(conf2);
120     utility2.setZkCluster(miniZK);
121     zkw2 = new ZooKeeperWatcher(conf2, "cluster2", null, true);
122 
123     admin.addPeer("2", utility2.getClusterKey());
124 
125     LOG.info("Setup second Zk");
126     CONF_WITH_LOCALFS = HBaseConfiguration.create(conf1);
127     utility1.startMiniCluster(2);
128     utility2.startMiniCluster(2);
129 
130     HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));
131     HColumnDescriptor fam = new HColumnDescriptor(famName);
132     fam.setMaxVersions(3);
133     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
134     table.addFamily(fam);
135     fam = new HColumnDescriptor(noRepfamName);
136     table.addFamily(fam);
137     HBaseAdmin admin1 = new HBaseAdmin(conf1);
138     HBaseAdmin admin2 = new HBaseAdmin(conf2);
139     admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
140     utility1.waitUntilAllRegionsAssigned(TableName.valueOf(tableName));
141     admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
142     utility2.waitUntilAllRegionsAssigned(TableName.valueOf(tableName));
143     htable1 = new HTable(conf1, tableName);
144     htable1.setWriteBufferSize(1024);
145     htable2 = new HTable(conf2, tableName);
146   }
147 
148   /**
149    * @throws java.lang.Exception
150    */
151   @AfterClass
152   public static void tearDownAfterClass() throws Exception {
153     utility2.shutdownMiniCluster();
154     utility1.shutdownMiniCluster();
155   }
156 
157 
158 }
159