Java Code Examples for org.apache.hadoop.hbase.replication.ReplicationPeerConfig#setClusterKey()

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 60000);
  TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 120000);
  TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
  TEST_UTIL.getConfiguration().setInt(START_LOG_ERRORS_AFTER_COUNT_KEY, 0);
  TEST_UTIL.startMiniCluster();
  ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get();

  conf2 = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  TEST_UTIL2 = new HBaseTestingUtility(conf2);
  TEST_UTIL2.startMiniCluster();
  admin2 =
      ConnectionFactory.createAsyncConnection(TEST_UTIL2.getConfiguration()).get().getAdmin();

  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL2.getClusterKey());
  ASYNC_CONN.getAdmin().addReplicationPeer(ID_SECOND, rpc).join();
}
 

@Test
public void testPeerConfig() throws Exception {
  ReplicationPeerConfig config = new ReplicationPeerConfig();
  config.setClusterKey(KEY_ONE);
  config.getConfiguration().put("key1", "value1");
  config.getConfiguration().put("key2", "value2");
  admin.addReplicationPeer(ID_ONE, config).join();

  List<ReplicationPeerDescription> peers = admin.listReplicationPeers().get();
  assertEquals(1, peers.size());
  ReplicationPeerDescription peerOne = peers.get(0);
  assertNotNull(peerOne);
  assertEquals("value1", peerOne.getPeerConfig().getConfiguration().get("key1"));
  assertEquals("value2", peerOne.getPeerConfig().getConfiguration().get("key2"));

  admin.removeReplicationPeer(ID_ONE).join();
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1 = HBaseConfiguration.create();
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setLong("replication.source.sleepforretries", 100);
  // Each WAL is about 120 bytes
  conf1.setInt(HConstants.REPLICATION_SOURCE_TOTAL_BUFFER_KEY, REPLICATION_SOURCE_QUOTA);
  conf1.setLong("replication.source.per.peer.node.bandwidth", 100L);

  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  new ZKWatcher(conf1, "cluster1", null, true);

  conf2 = new Configuration(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");

  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);
  new ZKWatcher(conf2, "cluster2", null, true);

  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(utility2.getClusterKey());

  utility1.startMiniCluster();
  utility2.startMiniCluster();

  Admin admin1 = ConnectionFactory.createConnection(conf1).getAdmin();
  admin1.addReplicationPeer("peer1", rpc);
  admin1.addReplicationPeer("peer2", rpc);
  admin1.addReplicationPeer("peer3", rpc);
  numOfPeer = admin1.listReplicationPeers().size();
}
 

@Test
public void testEnableDisablePeer() throws Exception {
  ReplicationPeerConfig rpc1 = new ReplicationPeerConfig();
  rpc1.setClusterKey(KEY_ONE);
  admin.addReplicationPeer(ID_ONE, rpc1).join();
  List<ReplicationPeerDescription> peers = admin.listReplicationPeers().get();
  assertEquals(1, peers.size());
  assertTrue(peers.get(0).isEnabled());

  admin.disableReplicationPeer(ID_ONE).join();
  peers = admin.listReplicationPeers().get();
  assertEquals(1, peers.size());
  assertFalse(peers.get(0).isEnabled());
  admin.removeReplicationPeer(ID_ONE).join();
}
 

@Test
public void testSetPeerNamespaces() throws Exception {
  String ns1 = "ns1";
  String ns2 = "ns2";

  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(KEY_ONE);
  admin.addReplicationPeer(ID_ONE, rpc).join();
  rpc.setReplicateAllUserTables(false);
  admin.updateReplicationPeerConfig(ID_ONE, rpc).join();

  // add ns1 and ns2 to peer config
  rpc = admin.getReplicationPeerConfig(ID_ONE).get();
  Set<String> namespaces = new HashSet<>();
  namespaces.add(ns1);
  namespaces.add(ns2);
  rpc.setNamespaces(namespaces);
  admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
  namespaces = admin.getReplicationPeerConfig(ID_ONE).get().getNamespaces();
  assertEquals(2, namespaces.size());
  assertTrue(namespaces.contains(ns1));
  assertTrue(namespaces.contains(ns2));

  // update peer config only contains ns1
  rpc = admin.getReplicationPeerConfig(ID_ONE).get();
  namespaces = new HashSet<>();
  namespaces.add(ns1);
  rpc.setNamespaces(namespaces);
  admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
  namespaces = admin.getReplicationPeerConfig(ID_ONE).get().getNamespaces();
  assertEquals(1, namespaces.size());
  assertTrue(namespaces.contains(ns1));

  admin.removeReplicationPeer(ID_ONE).join();
}
 

@Test
public void testPeerBandwidth() throws Exception {
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(KEY_ONE);

  admin.addReplicationPeer(ID_ONE, rpc).join();
  rpc = admin.getReplicationPeerConfig(ID_ONE).get();
  assertEquals(0, rpc.getBandwidth());

  rpc.setBandwidth(2097152);
  admin.updateReplicationPeerConfig(ID_ONE, rpc).join();
  assertEquals(2097152, admin.getReplicationPeerConfig(ID_ONE).join().getBandwidth());

  admin.removeReplicationPeer(ID_ONE).join();
}
 

@Before
public void setup() throws Exception {
  // setup configuration
  conf = HBaseConfiguration.create();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  setVisibilityLabelServiceImpl(conf);
  conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  VisibilityTestUtil.enableVisiblityLabels(conf);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      VisibilityReplication.class.getName());
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      SimpleCP.class.getName());
  // Have to reset conf1 in case zk cluster location different
  // than default
  conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
      ScanLabelGenerator.class);
  conf.set("hbase.superuser", User.getCurrent().getShortName());
  SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
      new String[] { "supergroup" });
  // User.createUserForTesting(conf, User.getCurrent().getShortName(), new
  // String[] { "supergroup" });
  USER1 = User.createUserForTesting(conf, "user1", new String[] {});
  TEST_UTIL = new HBaseTestingUtility(conf);
  TEST_UTIL.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
  zkw1 = new ZKWatcher(conf, "cluster1", null, true);

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf1 = HBaseConfiguration.create(conf);
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessorForTagsAtSink.class.getName());
  // setVisibilityLabelServiceImpl(conf1);
  USER1 = User.createUserForTesting(conf1, "user1", new String[] {});
  TEST_UTIL1 = new HBaseTestingUtility(conf1);
  TEST_UTIL1.setZkCluster(miniZK);
  zkw2 = new ZKWatcher(conf1, "cluster2", null, true);

  TEST_UTIL.startMiniCluster(1);
  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
  TEST_UTIL1.startMiniCluster(1);

  admin = TEST_UTIL.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL1.getClusterKey());
  admin.addReplicationPeer("2", rpc);

  Admin hBaseAdmin = TEST_UTIL.getAdmin();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try {
    hBaseAdmin.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin != null) {
      hBaseAdmin.close();
    }
  }
  Admin hBaseAdmin1 = TEST_UTIL1.getAdmin();
  try {
    hBaseAdmin1.createTable(tableDescriptor);
  } finally {
    if (hBaseAdmin1 != null) {
      hBaseAdmin1.close();
    }
  }
  addLabels();
  setAuths(conf);
  setAuths(conf1);
}
 

@Override
@Before
public void setup() throws Exception {
  expected[0] = 4;
  expected[1] = 6;
  expected[2] = 4;
  expected[3] = 0;
  expected[3] = 3;
  expectedVisString[0] = "(\"public\"&\"secret\"&\"topsecret\")|(\"confidential\"&\"topsecret\")";
  expectedVisString[1] = "(\"private\"&\"public\")|(\"private\"&\"topsecret\")|"
      + "(\"confidential\"&\"public\")|(\"confidential\"&\"topsecret\")";
  expectedVisString[2] = "(!\"topsecret\"&\"secret\")|(!\"topsecret\"&\"confidential\")";
  expectedVisString[3] = "(\"secret\"&\"" + COPYRIGHT + "\\\"" + ACCENT + "\\\\" + SECRET
      + "\\\"" + "\u0027&\\\\" + "\")";
  // setup configuration
  conf = HBaseConfiguration.create();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf.setInt("replication.source.size.capacity", 10240);
  conf.setLong("replication.source.sleepforretries", 100);
  conf.setInt("hbase.regionserver.maxlogs", 10);
  conf.setLong("hbase.master.logcleaner.ttl", 10);
  conf.setInt("zookeeper.recovery.retry", 1);
  conf.setInt("zookeeper.recovery.retry.intervalmill", 10);
  conf.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  conf.setInt("replication.stats.thread.period.seconds", 5);
  conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  setVisibilityLabelServiceImpl(conf, ExpAsStringVisibilityLabelServiceImpl.class);
  conf.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  VisibilityTestUtil.enableVisiblityLabels(conf);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      VisibilityReplication.class.getName());
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      SimpleCP.class.getName());
  // Have to reset conf1 in case zk cluster location different
  // than default
  conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class,
          ScanLabelGenerator.class);
  conf.set("hbase.superuser", "admin");
  conf.set("hbase.superuser", User.getCurrent().getShortName());
  SUPERUSER = User.createUserForTesting(conf, User.getCurrent().getShortName(),
      new String[] { "supergroup" });
  User.createUserForTesting(conf,
      User.getCurrent().getShortName(), new String[] { "supergroup" });
  USER1 = User.createUserForTesting(conf, "user1", new String[] {});
  TEST_UTIL = new HBaseTestingUtility(conf);
  TEST_UTIL.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster();
  zkw1 = new ZKWatcher(conf, "cluster1", null, true);

  // Base conf2 on conf1 so it gets the right zk cluster.
  conf1 = HBaseConfiguration.create(conf);
  conf1.setInt("hfile.format.version", 3);
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
  conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false);
  conf1.setStrings(HConstants.REPLICATION_CODEC_CONF_KEY, KeyValueCodecWithTags.class.getName());
  conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
          TestCoprocessorForTagsAtSink.class.getName());
  setVisibilityLabelServiceImpl(conf1, ExpAsStringVisibilityLabelServiceImpl.class);
  TEST_UTIL1 = new HBaseTestingUtility(conf1);
  TEST_UTIL1.setZkCluster(miniZK);
  zkw2 = new ZKWatcher(conf1, "cluster2", null, true);

  TEST_UTIL.startMiniCluster(1);
  // Wait for the labels table to become available
  TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
  TEST_UTIL1.startMiniCluster(1);

  admin = TEST_UTIL.getAdmin();
  ReplicationPeerConfig rpc = new ReplicationPeerConfig();
  rpc.setClusterKey(TEST_UTIL1.getClusterKey());
  admin.addReplicationPeer("2", rpc);

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  familyDescriptor.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  tableDescriptor.setColumnFamily(familyDescriptor);
  try (Admin hBaseAdmin = TEST_UTIL.getAdmin()) {
    hBaseAdmin.createTable(tableDescriptor);
  }
  try (Admin hBaseAdmin1 = TEST_UTIL1.getAdmin()){
    hBaseAdmin1.createTable(tableDescriptor);
  }
  addLabels();
  setAuths(conf);
  setAuths(conf1);
}