org.I0Itec.zkclient.exception.ZkTimeoutException Java Examples

/**
 * Connect to ZooKeeper.
 *
 * @param maxMsToWaitUntilConnected
 * @param watcher
 * @throws ZkInterruptedException
 *             if the connection timed out due to thread interruption
 * @throws ZkTimeoutException
 *             if the connection timed out
 * @throws IllegalStateException
 *             if the connection timed out due to thread interruption
 */
public void connect(final long maxMsToWaitUntilConnected, Watcher watcher) throws ZkInterruptedException, ZkTimeoutException, IllegalStateException {
    boolean started = false;
    acquireEventLock();
    try {
        setShutdownTrigger(false);
        _eventThread = new ZkEventThread(_connection.getServers());
        _eventThread.start();
        _connection.connect(watcher);

        LOG.debug("Awaiting connection to Zookeeper server");
        boolean waitSuccessful = waitUntilConnected(maxMsToWaitUntilConnected, TimeUnit.MILLISECONDS);
        if (!waitSuccessful) {
            throw new ZkTimeoutException("Unable to connect to zookeeper server '" + _connection.getServers() + "' with timeout of " + maxMsToWaitUntilConnected + " ms");
        }
        started = true;
    } finally {
        getEventLock().unlock();

        // we should close the zookeeper instance, otherwise it would keep
        // on trying to connect
        if (!started) {
            close();
        }
    }
}
 

@Override
public List<String> listTables(ExecutionContext context) throws ExecutorException {
  String address = environmentVariableHandler.getEnvironmentVariable(SAMZA_SQL_SYSTEM_KAFKA_ADDRESS);
  if (address == null || address.isEmpty()) {
    address = DEFAULT_SERVER_ADDRESS;
  }
  try {
    ZkUtils zkUtils = new ZkUtils(new ZkClient(address, DEFAULT_ZOOKEEPER_CLIENT_TIMEOUT),
        new ZkConnection(address), false);
    return JavaConversions.seqAsJavaList(zkUtils.getAllTopics())
      .stream()
      .map(x -> SAMZA_SYSTEM_KAFKA + "." + x)
      .collect(Collectors.toList());
  } catch (ZkTimeoutException ex) {
    throw new ExecutorException(ex);
  }
}
 

@Test(expectedExceptions = SiddhiAppValidationException.class,
        dependsOnMethods = "testKafkaMultipleTopicWithThreadingPerPartitionSource")
public void testKafkaWithoutThreadingOptionSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for without any threading option defined.");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"no_threading_option_topic"};
        KafkaTestUtil.createTopic(topics, 2);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='no_threading_option_topic', partition.no.list='0,1,2', "
                        + "group.id='test_no_threading_option_topic'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.start();
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(expectedExceptions = SiddhiAppValidationException.class,
        dependsOnMethods = "testKafkaMultipleTopic_MultiplePartition_AllPartitionSubscribe_Source")
public void testKafkaWithoutBootstrapServerSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for without any bootstrap servers defined.");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"no_bootstrap_server_topic"};
        KafkaTestUtil.createTopic(topics, 2);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='no_bootstrap_server_topic', partition.no.list='0,1,2', "
                        + "group.id='test', threading.option='single.thread'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.start();
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(expectedExceptions = SiddhiAppValidationException.class,
        dependsOnMethods = "testTransportCreationDisabledProperty")
public void testKafkaWithoutTopicSource() {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for without topic");
        log.info("-------------------------------------------------------------------------------------------");
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', group.id='test', threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.start();
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

/**
 * Connect to ZooKeeper.
 *
 * @param maxMsToWaitUntilConnected
 * @param watcher
 * @throws ZkInterruptedException
 *             if the connection timed out due to thread interruption
 * @throws ZkTimeoutException
 *             if the connection timed out
 * @throws IllegalStateException
 *             if the connection timed out due to thread interruption
 */
public void connect(final long maxMsToWaitUntilConnected, Watcher watcher) throws ZkInterruptedException, ZkTimeoutException, IllegalStateException {
    boolean started = false;
    acquireEventLock();
    try {
        setShutdownTrigger(false);
        _eventThread = new ZkEventThread(_connection.getServers());
        _eventThread.start();
        _connection.connect(watcher);

        LOG.debug("Awaiting connection to Zookeeper server");
        boolean waitSuccessful = waitUntilConnected(maxMsToWaitUntilConnected, TimeUnit.MILLISECONDS);
        if (!waitSuccessful) {
            throw new ZkTimeoutException("Unable to connect to zookeeper server '" + _connection.getServers() + "' with timeout of " + maxMsToWaitUntilConnected + " ms");
        }
        started = true;
    } finally {
        getEventLock().unlock();

        // we should close the zookeeper instance, otherwise it would keep
        // on trying to connect
        if (!started) {
            close();
        }
    }
}
 

/**
 * Create the connection with the zookeeper to consume the messages
 */
public boolean createKafkaConsumerConnector() throws Exception {

    log.debug("Create the connection and start to consume the streams");
    boolean isCreated;
    try {
        if (consumerConnector == null) {
            log.info("Creating Kafka Consumer Connector...");

            //set default consumer timeout to 3000ms if it is not set by the user
            if (!kafkaProperties.containsKey(KAFKAConstants.CONSUMER_TIMEOUT)) {
                kafkaProperties.put(KAFKAConstants.CONSUMER_TIMEOUT, "3000");
            }
            consumerConnector = Consumer.createJavaConsumerConnector(new ConsumerConfig(kafkaProperties));
            log.info("Kafka Consumer Connector is created");
            start();
        }
        isCreated = true;
    } catch (ZkTimeoutException toe) {
        log.error(" Error in Creating Kafka Consumer Connector | ZkTimeout" + toe.getMessage());
        throw new SynapseException(" Error in Creating Kafka Consumer Connector| ZkTimeout");

    } catch (Exception e) {
        log.error(" Error in Creating Kafka Consumer Connector." + e.getMessage(), e);
        throw new SynapseException(" Error in Creating Kafka Consumer Connector ", e);
    }
    return isCreated;
}
 

@Override
protected void startUp() throws Exception {
  int tries = 0;
  do {
    KafkaConfig kafkaConfig = createKafkaConfig(properties);
    KafkaServer kafkaServer = createKafkaServer(kafkaConfig);
    try {
      kafkaServer.startup();
      server = kafkaServer;
    } catch (Exception e) {
      kafkaServer.shutdown();
      kafkaServer.awaitShutdown();

      Throwable rootCause = Throwables.getRootCause(e);
      if (rootCause instanceof ZkTimeoutException) {
        // Potentially caused by race condition bug described in TWILL-139.
        LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause);
      } else if (rootCause instanceof BindException) {
        LOG.warn("Kafka failed to bind to port {}. Attempt number {}.", kafkaConfig.port(), tries, rootCause);
      } else {
        throw e;
      }

      // Do a random sleep of < 200ms
      TimeUnit.MILLISECONDS.sleep(new Random().nextInt(200) + 1L);
    }
  } while (server == null && ++tries < startTimeoutRetries);

  if (server == null) {
    throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts.");
  }
}
 

@Test(expectedExceptions = SiddhiAppValidationException.class,
        dependsOnMethods = "testKafkaWithoutThreadingOptionSource")
public void testKafkaSingleTopicWithoutGroupIdSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for single topic without group ID");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"single_topic_without_groupid"};
        KafkaTestUtil.createTopic(topics, 1);
        SiddhiManager siddhiManager = new SiddhiManager();

        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='single_topic_without_groupid', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");

        siddhiAppRuntime.start();
        KafkaTestUtil.deleteTopic(topics);
        Thread.sleep(4000);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaSingleTopicWithSpecificSubscribeSource")
public void testKafkaSpecificSubscribeForUnavailablePartitionSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for single topic with partitions which subscribes for an unavailable partition.");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"topic_without_some_partition"};
        KafkaTestUtil.createTopic(topics, 2);
        SiddhiManager siddhiManager = new SiddhiManager();
        Logger logger = Logger.getLogger(Source.class);
        UnitTestAppender appender = new UnitTestAppender();
        logger.addAppender(appender);
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='topic_without_some_partition', "
                        + "partition.no.list='0,1,2', "
                        + "group.id='test_topic_without_some_partition', threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.start();
        Thread.sleep(5000);
        if (appender.getMessages() != null) {
            AssertJUnit.assertTrue(appender.getMessages().contains("Error on 'TestExecutionPlan'. Partition " +
                    "number(s) 2 aren't available for the topic: topic_without_some_partition Error while " +
                    "connecting at Source 'kafka' at 'FooStream'."));
        }
        logger.removeAppender(appender);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaSpecificSubscribeForUnavailablePartitionSource")
public void testKafkaMultipleTopic_MultiplePartition_OnePartitionSubscribe_Source() throws
        InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
                + "subscribing for single partition id");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"multiple_topic1_two_par_one_sub", "multiple_topic2_two_par_one_sub"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 2);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='multiple_topic1_two_par_one_sub,multiple_topic2_two_par_one_sub', "
                        + "partition.no.list='1', "
                        + "group.id='test_multiple_topic1_two_par_one_sub', "
                        + "threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1_two_par_one_sub");
        expectedNames.add("multiple_topic2_two_par_one_sub");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(1L);
        expectedValues.add(1L);
        AssertJUnit.assertEquals(2, count);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaMultipleTopicPartitionTopicWiseSubscription")
public void testKafkaTopic_MultiplePartition_AllSubscribe_Source() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test to configure Kafka source with a single topic having multiple partitions "
                + "subscribing for all partition ids");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"kafka_topic_888"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 4);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='kafka_topic_888', "
                        + "partition.no.list='0,1,2,3', "
                        + "group.id='test_single_topic1_multiple_par_all_sub', "
                        + "threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 4, 4, true, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("kafka_topic_888");
        expectedNames.add("kafka_topic_888");
        expectedNames.add("kafka_topic_888");
        expectedNames.add("kafka_topic_888");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(2L);
        expectedValues.add(3L);
        AssertJUnit.assertEquals(4, count);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames, receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

public void testKafkaSingleTopic_MultiplePartition_AllPartitionSubscribe_Source_106() throws
        InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
                + "subscribing for all partition ids");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"single_topic1_two_par_two_servers"};
        KafkaTestUtil.setupKafkaBroker2();
        receivedEventNameList = new ArrayList<>(2);
        receivedValueList = new ArrayList<>(2);
        Thread.sleep(10000);
        KafkaTestUtil.createTopic(topics, 2);
        Thread.sleep(5000);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='single_topic1_two_par_two_servers', "
                        + "partition.no.list='0,1', "
                        + "group.id='test', "
                        + "threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092,localhost:9093'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        Thread.sleep(2000);
        KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, "localhost:9093,localhost:9092", true);
        Thread.sleep(1000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("single_topic1_two_par_two_servers");
        expectedNames.add("single_topic1_two_par_two_servers");
        expectedNames.add("single_topic1_two_par_two_servers");
        expectedNames.add("single_topic1_two_par_two_servers");
        List<Long> expectedValues = new ArrayList<>(2);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals(2, count);
        KafkaTestUtil.deleteTopic(topics);
        Thread.sleep(1000);
        KafkaTestUtil.stopKafkaBroker2();
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaMultipleTopicPartitionPartitionWiseSubscription")
public void testKafkaMultipleTopicPartitionTopicWiseSubscription() throws InterruptedException {
    log.info("-------------------------------------------------------------------------------------------");
    log.info("Creating test for multiple topics and partitions and thread topic wise");
    log.info("-------------------------------------------------------------------------------------------");
    try {
        String topics[] = new String[]{"kafka_topic_with_2par", "kafka_topic2_with_2par"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 2);
        Thread.sleep(100);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='kafka_topic_with_2par,kafka_topic2_with_2par', "
                        + "group.id='test_kafka_topic_with_2par_kafka_topic2_with_2par', "
                        + "threading.option='topic.wise', bootstrap.servers='localhost:9092', "
                        + "partition.no.list='0,1', " +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
        Thread.sleep(100);
        AssertJUnit.assertEquals(4, count);
        AssertJUnit.assertTrue(eventArrived);
        List<String> expectedNames = new ArrayList<>(4);
        expectedNames.add("kafka_topic_with_2par");
        expectedNames.add("kafka_topic_with_2par");
        expectedNames.add("kafka_topic2_with_2par");
        expectedNames.add("kafka_topic2_with_2par");
        List<Long> expectedValues = new ArrayList<>(4);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaSingleTopicSameGroupIdsSource")
public void testKafkaNonExistingTopicSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for non-existing topic. This will create a topic with default partition");
        log.info("-------------------------------------------------------------------------------------------");
        receivedEventNameList = new ArrayList<>(2);
        receivedValueList = new ArrayList<>(2);
        String topics[] = new String[]{"non_existing_topic1"};
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='non_existing_topic1', enable.auto.commit='false', "
                        + "group.id='test_non_existing_topic1', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 1, 1, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("non_existing_topic1");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        AssertJUnit.assertEquals(1, count);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaMultipleTopicWithThreadingPerTopicSource")
public void testKafkaMultipleTopicWithThreadingPerPartitionSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for multiple topic with thread per partition");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"multiple_topic1_2par", "multiple_topic2_2par"};
        receivedEventNameList = new ArrayList<>(2);
        receivedValueList = new ArrayList<>(2);
        KafkaTestUtil.createTopic(topics, 2);
        Thread.sleep(100);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='multiple_topic1_2par,multiple_topic2_2par', "
                        + "group.id='test_multiple_topic1_2par_multiple_topic2_2par', "
                        + "threading.option='partition.wise', "
                        + "bootstrap.servers='localhost:9092', "
                        + "partition.no.list='0,1'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1_2par");
        expectedNames.add("multiple_topic1_2par");
        expectedNames.add("multiple_topic2_2par");
        expectedNames.add("multiple_topic2_2par");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(4, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaWithoutBootstrapServerSource")
public void testKafkaMultipleTopicWithThreadingPerTopicSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for multiple topic with thread per topic");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
        receivedEventNameList = new ArrayList<>(2);
        receivedValueList = new ArrayList<>(2);
        KafkaTestUtil.createTopic(topics, 1);
        SiddhiManager siddhiManager = new SiddhiManager();

        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='multiple_topic1,multiple_topic2', "
                        + "group.id='test_multiple_topic1_multiple_topic2', "
                        + "threading.option='topic.wise', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        Thread.sleep(10000);
        KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
        Thread.sleep(300);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1");
        expectedNames.add("multiple_topic1");
        expectedNames.add("multiple_topic2");
        expectedNames.add("multiple_topic2");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);

        SiddhiTestHelper.waitForEvents(5000, 4, count, 15000);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(4, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test
public void testPublisherUsingBinaryMapper() throws InterruptedException {
    LOG.info("Creating test for publishing events using binary mapper.");
    String topics[] = new String[]{"single_topic"};
    KafkaTestUtil.createTopic(topics, 1);
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan1') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='single_topic', group.id='single_topic_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092', " +
                        "is.binary.message='true'," +
                        "@map(type='binary'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                EventPrinter.print(events);
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='single_topic', bootstrap.servers='localhost:9092', " +
                        "is.binary.message = 'true'," +
                        "@map(type='binary'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
        fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
        fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
        Thread.sleep(2000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("single_topic");
        expectedNames.add("single_topic2");
        expectedNames.add("single_topic3");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        expectedValues.add(103L);
        AssertJUnit.assertEquals("Kafka Sink  published the expected events", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Sink published the expected events", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(3, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaMultipleTopic_MultiplePartition_OnePartitionSubscribe_Source")
public void testKafkaMultipleTopic_MultiplePartition_AllPartitionSubscribe_Source() throws
        InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test to configure Kafka source with multiple topics having multiple partitions "
                + "subscribing for all partition ids");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"multiple_topic1_two_par_all_sub", "multiple_topic2_two_par_all_sub"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 2);
        Thread.sleep(1000);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='multiple_topic1_two_par_all_sub,multiple_topic2_two_par_all_sub', "
                        + "partition.no.list='0,1', "
                        + "group.id='test_multiple_topic1_two_par_all_sub', "
                        + "threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 2, 2, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1_two_par_all_sub");
        expectedNames.add("multiple_topic1_two_par_all_sub");
        expectedNames.add("multiple_topic2_two_par_all_sub");
        expectedNames.add("multiple_topic2_two_par_all_sub");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(4, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testPublisherWithTopicWithoutPartitionKafkaTransport")
public void testPublisherWithTopicWithPartitionKafkaTransport() throws InterruptedException {
    LOG.info("Creating test for publishing events for static topic with a partition");
    String topics[] = new String[]{"topic_with_two_partitions_sub0"};
    KafkaTestUtil.createTopic(topics, 2);
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan2') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='topic_with_two_partitions_sub0', partition.no.list='0',"
                        + "group.id='topic_with_two_partitions_sub0_test', threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan3') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='topic_with_two_partitions_sub0', partition.no='0', "
                        + "bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
        fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
        fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
        Thread.sleep(2000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("single_topic");
        expectedNames.add("single_topic2");
        expectedNames.add("single_topic3");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        expectedValues.add(103L);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(3, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaMultipleTopicSource")
public void testKafkaSingleTopicWithSpecificSubscribeSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for single topic with one partition which subscribes for the partition "
                + "specifically");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"topic_with_one_partition"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 1);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='topic_with_one_partition', "
                        + "partition.no.list='0', "
                        + "group.id='test_topic_with_one_partition', "
                        + "threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 1, 1, true, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("topic_with_one_partition");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(1, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testKafkaWithoutTopicSource")
public void testKafkaMultipleTopicSource() throws InterruptedException {
    try {
        log.info("-------------------------------------------------------------------------------------------");
        log.info("Creating test for multiple topic");
        log.info("-------------------------------------------------------------------------------------------");
        String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 1);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='multiple_topic1,multiple_topic2', "
                        + "group.id='test_multiple_topic1_multiple_topic2', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1");
        expectedNames.add("multiple_topic1");
        expectedNames.add("multiple_topic2");
        expectedNames.add("multiple_topic2");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);
        AssertJUnit.assertEquals(4, count);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test
public void testPublisherWithTopicWithoutPartitionKafkaTransport() throws InterruptedException {
    LOG.info("Creating test for publishing events for static topic without a partition");
    String topics[] = new String[]{"single_topic"};
    KafkaTestUtil.createTopic(topics, 1);
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan1') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='single_topic', group.id='single_topic_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='single_topic', bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        fooStream.send(new Object[]{"single_topic", 55.6f, 100L});
        fooStream.send(new Object[]{"single_topic2", 75.6f, 102L});
        fooStream.send(new Object[]{"single_topic3", 57.6f, 103L});
        Thread.sleep(3000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("single_topic");
        expectedNames.add("single_topic2");
        expectedNames.add("single_topic3");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        expectedValues.add(103L);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(3, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test
public void testKafkaSingleTopicSource() throws InterruptedException {
    try {
        log.info("Creating test for single topic");
        String topics[] = new String[]{"single_topic"};
        receivedEventNameList = new ArrayList<>(2);
        receivedValueList = new ArrayList<>(2);
        KafkaTestUtil.createTopic(topics, 1);
        Thread.sleep(1000);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') @App:transportChannelCreationEnabled('false')" +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='single_topic', group.id='test_single_topic', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        KafkaTestUtil.kafkaPublisher(topics, 1, 2, false, null, true);
        Thread.sleep(100);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("single_topic");
        expectedNames.add("single_topic");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        SiddhiTestHelper.waitForEvents(2000, 2, count, 20000);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(2, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test
public void testAKafkaPauseAndResume() throws InterruptedException {
    try {
        log.info("Test to verify the pause and resume functionality of Kafka source");
        String topics[] = new String[]{"kafka_topic3"};
        KafkaTestUtil.createTopic(topics, 2);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='kafka_topic3', group.id='test1', threading" +
                        ".option='partition.wise', " +
                        "bootstrap.servers='localhost:9092', partition.no.list='0,1', " +
                        "@map(type='text'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    log.info(event);
                    eventArrived = true;
                    count++;
                }

            }
        });
        siddhiAppRuntime.start();
        Future eventSender = executorService.submit(new Runnable() {
            @Override
            public void run() {
                KafkaTestUtil.kafkaPublisher(topics, 2, 4, false, null, false);
            }
        });
        while (!eventSender.isDone()) {
            Thread.sleep(1000);
        }
        Thread.sleep(2000);
        assertEquals(4, count);
        assertTrue(eventArrived);

        Collection<List<Source>> sources = siddhiAppRuntime.getSources();
        // pause the transports
        sources.forEach(e -> e.forEach(Source::pause));

        init2();
        eventSender = executorService.submit(new Runnable() {
            @Override
            public void run() {
                KafkaTestUtil.kafkaPublisher(topics, 2, 4, false, null, false);
            }
        });
        while (!eventSender.isDone()) {
            Thread.sleep(1000);
        }
        Thread.sleep(5000);
        assertFalse(eventArrived);

        // resume the transports
        sources.forEach(e -> e.forEach(Source::resume));
        Thread.sleep(2000);
        assertEquals(4, count);
        assertTrue(eventArrived);

        siddhiAppRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        log.warn("No zookeeper may not be available.", ex);
    }
}
 

public void testPublisherWithKafkaTransportWithDynamicTopicAndPartition() throws InterruptedException {
    LOG.info("Creating test for publishing events for dynamic topic with dynamic partition");
    try {
        String topics[] = new String[]{"multiple_topic1_two_par_all_sub1", "multiple_topic2_two_par_all_sub1"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 2);
        Thread.sleep(10000);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime executionPlanRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan14') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='{{symbol}}', partition.no='{{volume}}', "
                        + "bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream; ");
        InputHandler fooStream = executionPlanRuntime.getInputHandler("FooStream");
        executionPlanRuntime.start();


        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan15') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', "
                        + "topic.list='multiple_topic1_two_par_all_sub1,multiple_topic2_two_par_all_sub1', "
                        + "group.id='multiple_topic1_two_par_all_sub1_test', threading.option='single.thread', "
                        + "bootstrap.servers='localhost:9092', "
                        + "partition.no.list='0,1'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        Thread.sleep(2000);
        fooStream.send(new Object[]{"multiple_topic1_two_par_all_sub1", 55.6f, 0L});
        Thread.sleep(1000);
        fooStream.send(new Object[]{"multiple_topic1_two_par_all_sub1", 55.6f, 1L});
        Thread.sleep(1000);
        fooStream.send(new Object[]{"multiple_topic2_two_par_all_sub1", 75.6f, 0L});
        Thread.sleep(1000);
        fooStream.send(new Object[]{"multiple_topic2_two_par_all_sub1", 75.6f, 1L});
        Thread.sleep(5000);

        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1_two_par_all_sub1");
        expectedNames.add("multiple_topic1_two_par_all_sub1");
        expectedNames.add("multiple_topic2_two_par_all_sub1");
        expectedNames.add("multiple_topic2_two_par_all_sub1");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(0L);
        expectedValues.add(1L);
        expectedValues.add(0L);
        expectedValues.add(1L);
        AssertJUnit.assertEquals(4, count);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        executionPlanRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods =
        "testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransportReceiveMessage")
public void testPublisherWithKafkaTransportWithDynamicTopic() throws InterruptedException {
    LOG.info("Creating test for publishing events for dynamic topic without partition");
    try {
        String topics[] = new String[]{"multiple_topic1", "multiple_topic2"};
        receivedEventNameList = new ArrayList<>(4);
        receivedValueList = new ArrayList<>(4);
        KafkaTestUtil.createTopic(topics, 1);
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime executionPlanRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan12') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='{{symbol}}', bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream; ");
        InputHandler fooStream = executionPlanRuntime.getInputHandler("FooStream");
        executionPlanRuntime.start();


        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan13') " +
                        "define stream BarStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='multiple_topic1,multiple_topic2', "
                        + "group.id='multiple_topic1_multiple_topic2_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092'," +
                        "@map(type='xml'))" +
                        "Define stream FooStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        siddhiAppRuntime.addCallback("BarStream", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntime.start();
        fooStream.send(new Object[]{"multiple_topic1", 55.6f, 100L});
        fooStream.send(new Object[]{"multiple_topic2", 75.6f, 102L});
        Thread.sleep(2000);

        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("multiple_topic1");
        expectedNames.add("multiple_topic2");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        AssertJUnit.assertEquals(2, count);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Source expected input not received", expectedValues, receivedValueList);
        KafkaTestUtil.deleteTopic(topics);
        executionPlanRuntime.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

public void testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransport() throws InterruptedException {
    LOG.info("Creating test for publishing events for invalid topic with a partition other than 0 but the source "
            + "will be not be getting events from the default partition since the key is not defined");
    String topics[] = new String[]{"invalid_topic_with_partition_2"};
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan10') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='invalid_topic_with_partition_2', "
                        + "bootstrap.servers='localhost:9092', partition.no='2', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        Thread.sleep(2000);
        //this event will be published to the cluster and create the topic
        fooStream.send(new Object[]{"invalid_topic_with_partition_2", 55.6f, 100L});
        //this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
        // will create the topic.
        Thread.sleep(2000);
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan11') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='invalid_topic_with_partition_2', "
                        + "group.id='invalid_topic_with_partition_2_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        Thread.sleep(2000);
        fooStream.send(new Object[]{"invalid_topic_with_partition_22", 75.6f, 102L});
        fooStream.send(new Object[]{"invalid_topic_with_partition_23", 57.6f, 103L});
        Thread.sleep(2000);
        AssertJUnit.assertEquals(0, count);
        KafkaTestUtil.deleteTopic(topics);
        Thread.sleep(4000);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testPublisherWithInvalidTopicWithPartitionKafkaTransport")
public void testPublisherWithInvalidTopicWithPartitionOtherThan0KafkaTransportReceiveMessage() throws
        InterruptedException {
    LOG.info("Creating test for publishing events for invalid topic with a partition other than 0 but the source "
            + "will be getting events from the default partition when given a key");
    String topics[] = new String[]{"invalid_topic_with_partition_3"};
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan8') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='invalid_topic_with_partition_3', "
                        + "bootstrap.servers='localhost:9092', key='{{volume}}', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        //this event will be published to the cluster and create the topic
        fooStream.send(new Object[]{"invalid_topic_with_partition_2", 55.6f, 100L});
        //this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
        // will create the topic.
        Thread.sleep(500);
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan9') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='invalid_topic_with_partition_3', "
                        + "group.id='invalid_topic_with_partition_3_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092', " +
                        "@map(type='xml'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        fooStream.send(new Object[]{"invalid_topic_with_partition_22", 75.6f, 102L});
        fooStream.send(new Object[]{"invalid_topic_with_partition_23", 57.6f, 103L});
        Thread.sleep(2000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("invalid_topic_with_partition_2");
        expectedNames.add("invalid_topic_with_partition_22");
        expectedNames.add("invalid_topic_with_partition_23");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        expectedValues.add(103L);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(3, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}
 

@Test(dependsOnMethods = "testPublisherWithInvalidTopicKafkaTransport")
public void testPublisherWithInvalidTopicWithPartitionKafkaTransport() throws InterruptedException {
    LOG.info("Creating test for publishing events for invalid topic with a partition");
    String topics[] = new String[]{"invalid_topic_with_partition"};
    receivedEventNameList = new ArrayList<>(3);
    receivedValueList = new ArrayList<>(3);
    try {
        SiddhiManager siddhiManager = new SiddhiManager();
        SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan6') " +
                        "define stream FooStream (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@sink(type='kafka', topic='invalid_topic_with_partition', "
                        + "bootstrap.servers='localhost:9092', partition.no='0', " +
                        "@map(type='xml'))" +
                        "Define stream BarStream (symbol string, price float, volume long);" +
                        "from FooStream select symbol, price, volume insert into BarStream;");
        InputHandler fooStream = siddhiAppRuntime.getInputHandler("FooStream");
        siddhiAppRuntime.start();
        //this event will be published to the cluster and create the topic
        fooStream.send(new Object[]{"invalid_topic_with_partition", 55.6f, 100L});
        //this thread sleep is to slowdown the below execution plan because, if there are no topics kafka source
        // will create the topic.
        Thread.sleep(500);
        SiddhiAppRuntime siddhiAppRuntimeSource = siddhiManager.createSiddhiAppRuntime(
                "@App:name('TestExecutionPlan7') " +
                        "define stream BarStream2 (symbol string, price float, volume long); " +
                        "@info(name = 'query1') " +
                        "@source(type='kafka', topic.list='invalid_topic_with_partition', "
                        + "group.id='invalid_topic_with_partition_test', " +
                        "threading.option='single.thread', bootstrap.servers='localhost:9092', "
                        + "partition.no.list='0', " +
                        "@map(type='xml'))" +
                        "Define stream FooStream2 (symbol string, price float, volume long);" +
                        "from FooStream2 select symbol, price, volume insert into BarStream2;");
        siddhiAppRuntimeSource.addCallback("BarStream2", new StreamCallback() {
            @Override
            public void receive(Event[] events) {
                for (Event event : events) {
                    LOG.info(event);
                    eventArrived = true;
                    count++;
                    receivedEventNameList.add(event.getData(0).toString());
                    receivedValueList.add((long) event.getData(2));
                }
            }
        });
        siddhiAppRuntimeSource.start();
        fooStream.send(new Object[]{"invalid_topic_with_partition2", 75.6f, 102L});
        fooStream.send(new Object[]{"invalid_topic_with_partition3", 57.6f, 103L});
        Thread.sleep(2000);
        List<String> expectedNames = new ArrayList<>(2);
        expectedNames.add("invalid_topic_with_partition");
        expectedNames.add("invalid_topic_with_partition2");
        expectedNames.add("invalid_topic_with_partition3");
        List<Long> expectedValues = new ArrayList<>(2);
        expectedValues.add(100L);
        expectedValues.add(102L);
        expectedValues.add(103L);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedNames,
                receivedEventNameList);
        AssertJUnit.assertEquals("Kafka Sink didnt publish the expected events", expectedValues, receivedValueList);
        AssertJUnit.assertEquals(3, count);
        KafkaTestUtil.deleteTopic(topics);
        siddhiAppRuntime.shutdown();
        siddhiAppRuntimeSource.shutdown();
    } catch (ZkTimeoutException ex) {
        LOG.warn("No zookeeper may not be available.", ex);
    }
}