Java Code Examples for kafka.utils.ZkUtils#close()

/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 * @param topicProperties - The optional properties of the topics to create.
 */
public static void createTopics(
        final String zookeeperServers,
        final Set<String> topicNames,
        final int partitions,
        final int replicationFactor,
        final Optional<Properties> topicProperties) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);

    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for(final String topicName : topicNames) {
            if(!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, topicProperties.orElse(new Properties()), RackAwareMode.Disabled$.MODULE$);
            }
        }
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 

/**
 * Create an Atlas topic.
 *
 * The topic will get created based on following conditions:
 * {@link #ATLAS_NOTIFICATION_CREATE_TOPICS_KEY} is set to true.
 * The topic does not already exist.
 * Note that despite this, there could be multiple topic creation calls that happen in parallel because hooks
 * run in a distributed fashion. Exceptions are caught and logged by this method to prevent the startup of
 * the hooks from failing.
 * @param atlasProperties {@link Configuration} containing properties to be used for creating topics.
 * @param topicNames list of topics to create
 */
public void createAtlasTopic(Configuration atlasProperties, String... topicNames) {
    if (atlasProperties.getBoolean(ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)) {
        if (!handleSecurity(atlasProperties)) {
            return;
        }
        ZkUtils zkUtils = createZkUtils(atlasProperties);
        for (String topicName : topicNames) {
            try {
                LOG.warn("Attempting to create topic {}", topicName);
                if (!ifTopicExists(topicName, zkUtils)) {
                    createTopic(atlasProperties, topicName, zkUtils);
                } else {
                    LOG.warn("Ignoring call to create topic {}, as it already exists.", topicName);
                }
            } catch (Throwable t) {
                LOG.error("Failed while creating topic {}", topicName, t);
            }
        }
        zkUtils.close();
    } else {
        LOG.info("Not creating topics {} as {} is false", StringUtils.join(topicNames, ","),
                ATLAS_NOTIFICATION_CREATE_TOPICS_KEY);
    }
}
 

/**
 * Create an Atlas topic.
 *
 * The topic will get created based on following conditions:
 * {@link #ATLAS_NOTIFICATION_CREATE_TOPICS_KEY} is set to true.
 * The topic does not already exist.
 * Note that despite this, there could be multiple topic creation calls that happen in parallel because hooks
 * run in a distributed fashion. Exceptions are caught and logged by this method to prevent the startup of
 * the hooks from failing.
 * @param atlasProperties {@link Configuration} containing properties to be used for creating topics.
 * @param topicNames list of topics to create
 */
public void createAtlasTopic(Configuration atlasProperties, String... topicNames) {
    if (atlasProperties.getBoolean(ATLAS_NOTIFICATION_CREATE_TOPICS_KEY, true)) {
        if (!handleSecurity(atlasProperties)) {
            return;
        }
        ZkUtils zkUtils = createZkUtils(atlasProperties);
        for (String topicName : topicNames) {
            try {
                LOG.warn("Attempting to create topic {}", topicName);
                if (!ifTopicExists(topicName, zkUtils)) {
                    createTopic(atlasProperties, topicName, zkUtils);
                } else {
                    LOG.warn("Ignoring call to create topic {}, as it already exists.", topicName);
                }
            } catch (Throwable t) {
                LOG.error("Failed while creating topic {}", topicName, t);
            }
        }
        zkUtils.close();
    } else {
        LOG.info("Not creating topics {} as {} is false", StringUtils.join(topicNames, ","),
                ATLAS_NOTIFICATION_CREATE_TOPICS_KEY);
    }
}
 

@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 

@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 

@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 

@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata();
			firstPart = partitionMetadata.head();
		}
		while (firstPart.errorCode() != 0);

		return firstPart.leader().get().id();
	} finally {
		zkUtils.close();
	}
}
 

@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 

@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata();
			firstPart = partitionMetadata.head();
		}
		while (firstPart.errorCode() != 0);

		return firstPart.leader().get().id();
	} finally {
		zkUtils.close();
	}
}
 

public static void checkAndCreateTopic(String zkConnect, String topic, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (AdminUtils.topicExists(zkUtils, topic)) {
    verifyTopic(zkUtils, topic);
    return;
  }

  int partitions = 1;
  Properties topicConfig = new Properties();
  topicConfig.put(LogConfig.CleanupPolicyProp(), "compact");

  AdminUtils.createTopic(zkUtils, topic, partitions, replicas, topicConfig, RackAwareMode.Enforced$.MODULE$);

  zkClient.close();
  zkUtils.close();
}
 

void createTopic(String topicName) {
    ZkUtils zu = null;
    try {
        zu = util.getUtils();
        AdminUtils.createTopic(zu, topicName, 2, 3, new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        if (zu != null) {
            zu.close();
        }
    }
}
 

@Test
public void testGetUtil() throws InterruptedException {
    ZkUtils u = null;
    try {
        u = util.getUtils();
        assertNotNull(u);
        assertNotNull(u.ClusterId());
    } finally {
        if (u != null) {
            u.close();
        }
    }
}
 

List<String> listTopics() throws Exception {
    ZkUtils zu = null;
    try {
        zu = util.getUtils();
        return JavaConversions.asJavaList(zu.getAllTopics());
    } finally {
        if (zu != null) {
            zu.close();
        }
    }
}
 

/**
 * 删除一个topic，这个删除只是告知系统标记该topic要被删除。而不是立即删除。
 * @param topicName topic的名字
 */
public static void deleteTopic(String topicName) {
    ZkUtils zkUtils = ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, JaasUtils.isZkSecurityEnabled());
    // 删除topic 't1'
    AdminUtils.deleteTopic(zkUtils, topicName);
    zkUtils.close();
}
 

private boolean topicExists(String topic) {
  ZkClient zkClient = new ZkClient(cluster.zKConnectString(), SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(cluster.zKConnectString()), IS_SECURE_KAFKA_CLUSTER);
  boolean exists = AdminUtils.topicExists(zkUtils, topic);
  zkClient.close();
  zkUtils.close();
  return exists;
}
 

@Test
public void compact_road_created_correctly() throws Exception {
  KafkaRoad model = new KafkaRoad("test_topic6", "road.test_topic6", RoadType.COMPACT, null, null, false);

  context.getBean(TrafficControl.class).newModel("test_topic6", model);

  ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false);
  Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "road.test_topic6");
  zkUtils.close();

  assertThat(config.getProperty(TopicConfig.CLEANUP_POLICY_CONFIG), is(TopicConfig.CLEANUP_POLICY_COMPACT));
}
 

public static boolean createKafkaTopicIfNecessary(String brokerUri, int replFactor, int numPartitions, String topic)
    throws IOException {
  URI zkUri = URI.create(brokerUri);
  Preconditions.checkArgument("zk".equals(zkUri.getScheme()));
  String zkServerList = zkUri.getAuthority() + zkUri.getPath();

  ZkUtils zkUtils = ZkUtils.apply(zkServerList, ZK_SESSION_TIMEOUT_MS,
      ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled());
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      return false;
    }

    try {
      AdminUtils.createTopic(zkUtils, topic, numPartitions, replFactor, new Properties());
    } catch (TopicExistsException ignored) {
      return false;
    } catch (RuntimeException e) {
      throw new IOException(e);
    }
  } finally {
    if (zkUtils != null) {
      zkUtils.close();
    }
  }
  return true;
}
 

public static void createTopics(String zookeeperUrl, int partitions, int replicas, String... topicNames) throws InterruptedException {

        ZkClient zkClient = ZkUtils.createZkClient(zookeeperUrl, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS);
        ZkConnection zkConnection = new ZkConnection(zookeeperUrl);
        ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);

        for (String topicName : topicNames) {
            AdminUtils.createTopic(zkUtils, topicName, partitions, replicas, new Properties(), RackAwareMode.Enforced$.MODULE$);
        }

        zkUtils.close();
        zkConnection.close();
        zkClient.close();
    }
 

/**
 * 创建kafka队列
 * @param topicName 队列名称
 * @param partitions 分区数量
 * @param replicationFactor topic中数据的副本数量，默认为3
 *
 */
public static void createTopic(String topicName, int partitions, int replicationFactor){

    ZkUtils zkUtils = ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, JaasUtils.isZkSecurityEnabled());
    // 创建一个单分区单副本名为t1的topic
    AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, new Properties(), RackAwareMode.Enforced$.MODULE$);
    zkUtils.close();

}
 

/**
 * Utility method to provide additional unique topics if they are required.
 * @param topicName - The Kafka topic to create.
 */
public void createTopic(final String topicName) {
    // Setup Kafka.
    ZkUtils zkUtils = null;
    try {
        logger.info("Creating Kafka Topic: '{}'", topicName);
        zkUtils = ZkUtils.apply(new ZkClient(kafkaInstance.getZookeeperConnect(), 30000, 30000, ZKStringSerializer$.MODULE$), false);
        AdminUtils.createTopic(zkUtils, topicName, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}