kafka.utils.Time Java Examples

The following examples show how to use kafka.utils.Time. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: EmbeddedKafkaServer.java    From twill with Apache License 2.0 6 votes vote down vote up 
private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
  return new KafkaServer(kafkaConfig, new Time() {

    @Override
    public long milliseconds() {
      return System.currentTimeMillis();
    }

    @Override
    public long nanoseconds() {
      return System.nanoTime();
    }

    @Override
    public void sleep(long ms) {
      try {
        Thread.sleep(ms);
      } catch (InterruptedException e) {
        Thread.interrupted();
      }
    }
  });
}
Example #2
Source File: EmbeddedKafkaInstance.java    From rya with Apache License 2.0 6 votes vote down vote up 
/**
 * Starts the Embedded Kafka and Zookeeper Servers.
 * @throws Exception - If an exeption occurs during startup.
 */
protected void startup() throws Exception {
    // Setup the embedded zookeeper
    logger.info("Starting up Embedded Zookeeper...");
    zkServer = new EmbeddedZookeeper();
    zookeperConnect = ZKHOST + ":" + zkServer.port();
    logger.info("Embedded Zookeeper started at: {}", zookeperConnect);

    // setup Broker
    logger.info("Starting up Embedded Kafka...");
    brokerPort = Integer.toString(PortUtils.getRandomFreePort());
    final Properties brokerProps = new Properties();
    brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0");
    brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST);
    brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort);
    brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect);
    brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString());
    brokerProps.setProperty(KafkaConfig$.MODULE$.DeleteTopicEnableProp(), "true");
    final KafkaConfig config = new KafkaConfig(brokerProps);
    final Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
    logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort);
}
Example #3
Source File: KafkaExportITBase.java    From rya with Apache License 2.0 6 votes vote down vote up 
/**
 * setup mini kafka and call the super to setup mini fluo
 */
@Before
public void setupKafka() throws Exception {
    // Install an instance of Rya on the Accumulo cluster.
    installRyaInstance();

    // Setup Kafka.
    zkServer = new EmbeddedZookeeper();
    final String zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    final Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    final KafkaConfig config = new KafkaConfig(brokerProps);
    final Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
}
Example #4
Source File: KafkaTestBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up 
public static void startServer() throws RuntimeException {
  if (serverStarted && serverClosed) {
    throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice.");
  }

  if (!serverStarted) {
    serverStarted = true;
    zkConnect = TestZKUtils.zookeeperConnect();
    zkServer = new EmbeddedZookeeper(zkConnect);
    zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$);

    kafkaPort = TestUtils.choosePort();
    Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true);

    KafkaConfig config = new KafkaConfig(props);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
  }
}
Example #5
Source File: KafkaTestBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up 
void start()
  throws RuntimeException {
  if (_numStarted.incrementAndGet() == 1) {
    log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort);
    _zkServer = new EmbeddedZookeeper(_zkConnectString);
    _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$);


    Properties props = kafka.utils.TestUtils.createBrokerConfig(_brokerId, _kafkaServerPort, true);
    props.setProperty("zookeeper.connect", _zkConnectString);

    KafkaConfig config = new KafkaConfig(props);
    Time mock = new MockTime();
    _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
  }
  else
  {
    log.info("Kafka server suite already started... continuing");
  }
}
Example #6
Source File: KafkaClusterTestBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up 
private KafkaServer createKafkaServer(int brokerId,String _zkConnectString){
	
	int _brokerId = brokerId;
	int _kafkaServerPort = TestUtils.findFreePort();	
	Properties props = kafka.utils.TestUtils.createBrokerConfig(
         _brokerId,
         _zkConnectString,
         kafka.utils.TestUtils.createBrokerConfig$default$3(),
         kafka.utils.TestUtils.createBrokerConfig$default$4(),
         _kafkaServerPort,
         kafka.utils.TestUtils.createBrokerConfig$default$6(),
         kafka.utils.TestUtils.createBrokerConfig$default$7(),
         kafka.utils.TestUtils.createBrokerConfig$default$8(),
         kafka.utils.TestUtils.createBrokerConfig$default$9(),
         kafka.utils.TestUtils.createBrokerConfig$default$10(),
         kafka.utils.TestUtils.createBrokerConfig$default$11(),
         kafka.utils.TestUtils.createBrokerConfig$default$12(),
         kafka.utils.TestUtils.createBrokerConfig$default$13(),
         kafka.utils.TestUtils.createBrokerConfig$default$14()
         );
     KafkaConfig config = new KafkaConfig(props);
     Time mock = new MockTime();
     KafkaServer _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
     kafkaBrokerPortList.add(_kafkaServerPort);
     return _kafkaServer;
}
Example #7
Source File: EmbeddedKafka.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up 
public void start() throws IOException
{
  // Find port
  try {
    ServerSocket serverSocket = new ServerSocket(0);
    BROKERPORT = Integer.toString(serverSocket.getLocalPort());
    serverSocket.close();
  } catch (IOException e) {
    throw Throwables.propagate(e);
  }

  // Setup Zookeeper
  zkServer = new EmbeddedZookeeper();
  String zkConnect = BROKERHOST + ":" + zkServer.port();
  zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
  zkUtils = ZkUtils.apply(zkClient, false);

  // Setup brokers
  cleanupDir();
  Properties props = new Properties();
  props.setProperty("zookeeper.connect", zkConnect);
  props.setProperty("broker.id", "0");
  props.setProperty("log.dirs", KAFKA_PATH);
  props.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
  KafkaConfig config = new KafkaConfig(props);
  Time mock = new MockTime();
  kafkaServer = TestUtils.createServer(config, mock);
}
Example #8
Source File: KafkaComponent.java    From metron with Apache License 2.0 5 votes vote down vote up 
@Override
public void start() {
  // setup Zookeeper
  zookeeperConnectString = topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY);

  zkClient = new ZkClient(zookeeperConnectString, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);

  // setup Broker
  Properties props = TestUtilsWrapper.createBrokerConfig(0, zookeeperConnectString, brokerPort);
  props.setProperty("zookeeper.connection.timeout.ms", Integer.toString(KAFKA_ZOOKEEPER_TIMEOUT_MS));
  KafkaConfig config = new KafkaConfig(props);
  Time mock = new MockTime();
  kafkaServer = TestUtils.createServer(config, mock);

  org.apache.log4j.Level oldLevel = UnitTestHelper.getLog4jLevel(KafkaServer.class);
  UnitTestHelper.setLog4jLevel(KafkaServer.class, org.apache.log4j.Level.OFF);
  // do not proceed until the broker is up
  TestUtilsWrapper.waitUntilBrokerIsRunning(kafkaServer,"Timed out waiting for RunningAsBroker State",100000);

  for(Topic topic : getTopics()) {
    try {
      createTopic(topic.name, topic.numPartitions, KAFKA_PROPAGATE_TIMEOUT_MS);
    } catch (InterruptedException e) {
      throw new RuntimeException("Unable to create topic", e);
    }
  }
  UnitTestHelper.setLog4jLevel(KafkaServer.class, oldLevel);
  if(postStartCallback != null) {
    postStartCallback.apply(this);
  }
}
Example #9
Source File: KafkaTestBase.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up 
void start()
  throws RuntimeException {
  if (_numStarted.incrementAndGet() == 1) {
    log.warn("Starting up Kafka server suite. Zk at " + _zkConnectString + "; Kafka server at " + _kafkaServerPort);
    _zkServer = new EmbeddedZookeeper();
    _zkConnectString = "127.0.0.1:"+_zkServer.port();
    _zkClient = new ZkClient(_zkConnectString, 30000, 30000, ZKStringSerializer$.MODULE$);

    Properties props = kafka.utils.TestUtils.createBrokerConfig(
        _brokerId,
        _zkConnectString,
        kafka.utils.TestUtils.createBrokerConfig$default$3(),
        kafka.utils.TestUtils.createBrokerConfig$default$4(),
        _kafkaServerPort,
        kafka.utils.TestUtils.createBrokerConfig$default$6(),
        kafka.utils.TestUtils.createBrokerConfig$default$7(),
        kafka.utils.TestUtils.createBrokerConfig$default$8(),
        kafka.utils.TestUtils.createBrokerConfig$default$9(),
        kafka.utils.TestUtils.createBrokerConfig$default$10(),
        kafka.utils.TestUtils.createBrokerConfig$default$11(),
        kafka.utils.TestUtils.createBrokerConfig$default$12(),
        kafka.utils.TestUtils.createBrokerConfig$default$13(),
        kafka.utils.TestUtils.createBrokerConfig$default$14()
        );


    KafkaConfig config = new KafkaConfig(props);
    Time mock = new MockTime();
    _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
  }
  else
  {
    log.info("Kafka server suite already started... continuing");
  }
}