Java Code Examples for kafka.utils.ZkUtils#apply()
The following examples show how to use
kafka.utils.ZkUtils#apply() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaProducer09IT.java From datacollector with Apache License 2.0 | 6 votes |
@BeforeClass public static void setUpClass() throws Exception { int zkConnectionTimeout = 6000; int zkSessionTimeout = 6000; zookeeper = new EmbeddedZookeeper(); zkConnect = String.format("127.0.0.1:%d", zookeeper.port()); zkUtils = ZkUtils.apply( zkConnect, zkSessionTimeout, zkConnectionTimeout, JaasUtils.isZkSecurityEnabled()); port = NetworkUtils.getRandomPort(); kafkaServer = TestUtil09.createKafkaServer(port, zkConnect); for (int i = 0; i < topics.length; i++) { topics[i] = UUID.randomUUID().toString(); AdminUtils.createTopic(zkUtils, topics[i], 1, 1, new Properties()); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(Arrays.asList(kafkaServer)), topics[i], 0, 5000); } }
Example 2
Source File: KafkaUtilsTest.java From incubator-samoa with Apache License 2.0 | 6 votes |
@BeforeClass public static void setUpClass() throws IOException { // setup Zookeeper zkServer = new EmbeddedZookeeper(); zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); ZkUtils zkUtils = ZkUtils.apply(zkClient, false); // setup Broker Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); KafkaConfig config = new KafkaConfig(brokerProps); Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); // create topics AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$); }
Example 3
Source File: Topicutil.java From dk-fitting with Apache License 2.0 | 6 votes |
/** * 查询所有topic,包括已经被标记删除,还没有删除的topic。 * @return topic的list */ public static List<String> queryAllTopic(){ ZkUtils zkUtils = ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, JaasUtils.isZkSecurityEnabled()); ArrayList<String> topics = new ArrayList<String>(); // AdminUtils.topicExists() scala.collection.Map<String, Properties> stringPropertiesMap = AdminUtils.fetchAllTopicConfigs(zkUtils); Map<String, Properties> javaMap = JavaConversions.mapAsJavaMap(stringPropertiesMap); Iterator<String> iterator = javaMap.keySet().iterator(); while(iterator.hasNext()){ String key = iterator.next(); Properties properties = javaMap.get(key); topics.add(key); } zkUtils.close(); return topics; }
Example 4
Source File: KafkaComponent.java From metron with Apache License 2.0 | 6 votes |
public void createTopic(String name, int numPartitions, long waitThisLongForMetadataToPropagate) throws InterruptedException { ZkUtils zkUtils = null; Level oldLevel = UnitTestHelper.getJavaLoggingLevel(); try { UnitTestHelper.setJavaLoggingLevel(Level.OFF); zkUtils = ZkUtils.apply(zookeeperConnectString, 30000, 30000, false); AdminUtilsWrapper.createTopic(zkUtils, name, numPartitions, 1, new Properties()); if (waitThisLongForMetadataToPropagate > 0) { waitUntilMetadataIsPropagated(name, numPartitions, waitThisLongForMetadataToPropagate); } }catch(TopicExistsException tee) { }finally { if(zkUtils != null){ zkUtils.close(); } UnitTestHelper.setJavaLoggingLevel(oldLevel); } }
Example 5
Source File: KafkaExportITBase.java From rya with Apache License 2.0 | 6 votes |
/** * setup mini kafka and call the super to setup mini fluo */ @Before public void setupKafka() throws Exception { // Install an instance of Rya on the Accumulo cluster. installRyaInstance(); // Setup Kafka. zkServer = new EmbeddedZookeeper(); final String zkConnect = ZKHOST + ":" + zkServer.port(); zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$); zkUtils = ZkUtils.apply(zkClient, false); // setup Broker final Properties brokerProps = new Properties(); brokerProps.setProperty("zookeeper.connect", zkConnect); brokerProps.setProperty("broker.id", "0"); brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString()); brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT); final KafkaConfig config = new KafkaConfig(brokerProps); final Time mock = new MockTime(); kafkaServer = TestUtils.createServer(config, mock); }
Example 6
Source File: KafkaUtils.java From oryx with Apache License 2.0 | 5 votes |
/** * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...] * @param topic topic to delete, if it exists */ public static void deleteTopic(String zkServers, String topic) { ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false); try { if (AdminUtils.topicExists(zkUtils, topic)) { log.info("Deleting topic {}", topic); AdminUtils.deleteTopic(zkUtils, topic); log.info("Deleted Zookeeper topic {}", topic); } else { log.info("No need to delete topic {} as it does not exist", topic); } } finally { zkUtils.close(); } }
Example 7
Source File: Utils.java From kafka-monitor with Apache License 2.0 | 5 votes |
public static int getPartitionNumByTopic(String zk,String topic){ ZkUtils zkUtils=ZkUtils.apply(zk,ZK_SESSION_TIMEOUT_MS,ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled()); try { return zkUtils.getPartitionsForTopics(JavaConversions.asScalaBuffer(Arrays.asList(topic))).apply(topic).size(); }finally { zkUtils.close(); } }
Example 8
Source File: KafkaValidationUtil09IT.java From datacollector with Apache License 2.0 | 5 votes |
@BeforeClass public static void setUp() throws IOException { int zkConnectionTimeout = 6000; int zkSessionTimeout = 6000; zookeeper = new EmbeddedZookeeper(); zkConnect = String.format("127.0.0.1:%d", zookeeper.port()); zkUtils = ZkUtils.apply( zkConnect, zkSessionTimeout, zkConnectionTimeout, JaasUtils.isZkSecurityEnabled()); port = NetworkUtils.getRandomPort(); kafkaServer = TestUtil09.createKafkaServer(port, zkConnect, false); sdcKafkaValidationUtil = SdcKafkaValidationUtilFactory.getInstance().create(); }
Example 9
Source File: TrafficControlApp.java From data-highway with Apache License 2.0 | 5 votes |
@Bean(destroyMethod = "close") public ZkUtils zkUtils( @Value("${kafka.zookeeper}") String zkUrl, @Value("${kafka.sessionTimeout:60000}") int sessionTimeout, @Value("${kafka.connectionTimeout:60000}") int connectionTimeout, @Value("${kafka.zkSecurityEnabled:false}") boolean zkSecurityEnabled) { return ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, zkSecurityEnabled); }
Example 10
Source File: KafkaTestEnvironmentImpl.java From flink with Apache License 2.0 | 4 votes |
public ZkUtils getZkUtils() { ZkClient creator = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); return ZkUtils.apply(creator, false); }
Example 11
Source File: KafkaTestEnvironmentImpl.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public ZkUtils getZkUtils() { ZkClient creator = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); return ZkUtils.apply(creator, false); }
Example 12
Source File: KafkaMessageNewReceiverPoolTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void setUp() throws Exception { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); KafkaConfig config = new KafkaConfig(kafkaProps); Time mock = new SystemTime(); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "2"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 13
Source File: KafkaMessageSenderImplTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "1"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 14
Source File: TestConfig.java From metron with Apache License 2.0 | 4 votes |
@Bean public ZkUtils zkUtils(ZkClient zkClient) { return ZkUtils.apply(zkClient, false); }
Example 15
Source File: ZookeeperBrokersTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "4"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 16
Source File: ZookeeperHostsTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "1"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 17
Source File: KafkaTestEnvironmentImpl.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
public ZkUtils getZkUtils() { LOG.info("In getZKUtils:: zookeeperConnectionString = {}", zookeeperConnectionString); ZkClient creator = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")), Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer()); return ZkUtils.apply(creator, false); }
Example 18
Source File: NewSenderWithSpringTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "1"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 19
Source File: ReceiverWithSpringTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "1"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }
Example 20
Source File: SenderTest.java From message-queue-client-framework with Apache License 2.0 | 4 votes |
@Before public void before() { try { zkServer = new EmbeddedZookeeper(); zkConnect = String.format("localhost:%d", zkServer.port()); ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000, JaasUtils.isZkSecurityEnabled()); zkClient = zkUtils.zkClient(); Time mock = new SystemTime(); final Option<File> noFile = scala.Option.apply(null); final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null); final Option<Properties> noPropertiesOption = scala.Option.apply(null); final Option<String> noStringOption = scala.Option.apply(null); kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false, false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true, false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort()); kafkaProps.setProperty("auto.create.topics.enable", "true"); kafkaProps.setProperty("num.partitions", "1"); // We *must* override this to use the port we allocated (Kafka currently // allocates one port // that it always uses for ZK kafkaProps.setProperty("zookeeper.connect", this.zkConnect); kafkaProps.setProperty("host.name", "localhost"); kafkaProps.setProperty("port", port + ""); KafkaConfig config = new KafkaConfig(kafkaProps); kafkaServer = TestUtils.createServer(config, mock); // create topic TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions( new String[]{"--create", "--topic", topic, "--replication-factor", "1", "--partitions", "1"}); TopicCommand.createTopic(zkUtils, options); List<KafkaServer> servers = new ArrayList<KafkaServer>(); servers.add(kafkaServer); TestUtils.waitUntilMetadataIsPropagated( scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000); } catch (Exception e) { } }