Java Code Examples for io.siddhi.core.SiddhiManager#setPersistenceStore()

@Test
public void persistenceTest1() throws InterruptedException {
    log.info("persistence test 1 - window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream" +
            "   #log()" +
            "   #log('test message')" +
            "   #log(false)" +
            "   #log(true)" +
            "   #log('test message',false)" +
            "   #log('test message',true)" +
            "   #log('error','test message')" +
            "   #log('error','test message',false)" +
            "   #log('warn','test message',true)" +
            "select * " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Event[]{
            new Event(System.currentTimeMillis(), new Object[]{"IBM", 75.6f, 100}),
            new Event(System.currentTimeMillis(), new Object[]{"GOOG", 70.6f, 100})});
    Thread.sleep(100);

    siddhiAppRuntime.shutdown();
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test(dependsOnMethods = "persistenceTest12")
public void persistenceTest13() throws InterruptedException {
    log.info("Persistence test 13 - partitioned sum with group-by on length windows.");
    final int inputEventCount = 10;
    PersistenceStore persistenceStore = new InMemoryPersistenceStore();
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "@app:name('incrementalPersistenceTest10') "
            + "define stream cseEventStreamOne (symbol string, price float,volume int);"
            + "partition with (price>=100 as 'large' or price<100 as 'small' of cseEventStreamOne) " +
            "begin " +
            "@info(name = 'query1') " +
            "from cseEventStreamOne#window.length(4) " +
            "select symbol,sum(price) as price " +
            "group by symbol " +
            "insert into OutStockStream;  " +
            "end ";


    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    StreamCallback streamCallback = new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);
            eventArrived = true;
            if (events != null) {
                for (Event event : events) {
                    count++;
                    lastValue = ((Double) event.getData(1)).longValue();
                }
            }
        }
    };

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 95f + i, 100});
        Thread.sleep(100);
        siddhiAppRuntime.persist();
    }

    inputHandler.send(new Object[]{"IBM", 205f, 100});
    Thread.sleep(100);

    siddhiAppRuntime.shutdown();

    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    siddhiAppRuntime.start();

    Thread.sleep(1000);

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    Thread.sleep(1000);

    inputHandler.send(new Object[]{"IBM", 105f, 100});

    Thread.sleep(1000);

    AssertJUnit.assertEquals(new Long(414), lastValue);

    siddhiAppRuntime.shutdown();
}
 

@Test(dependsOnMethods = "persistenceTest11")
public void persistenceTest12() throws InterruptedException {
    log.info("persistence test 12 - partition query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "@App:name('TestPlan1')\n" +
            "define stream TempStream(deviceID long);\n" +
            "\n" +
            "define stream DeviceTempStream (deviceID long, count long);\n" +
            "\n" +
            "from TempStream\n" +
            "select * insert into TempInternalStream;\n" +
            "\n" +
            "partition with ( deviceID of TempInternalStream )\n" +
            "begin\n" +
            "from TempInternalStream\n" +
            "select deviceID, count() as count\n" +
            "insert into DeviceTempStream\n" +
            "end;";

    StreamCallback queryCallback = new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);
            eventArrived = true;
            count++;
            lastValue = (Long) events[0].getData(1);
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("DeviceTempStream", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("TempStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{1});
    Thread.sleep(100);
    inputHandler.send(new Object[]{1});
    Thread.sleep(100);
    inputHandler.send(new Object[]{1});
    Thread.sleep(100);
    inputHandler.send(new Object[]{2});
    Thread.sleep(100);
    inputHandler.send(new Object[]{2});

    Thread.sleep(600);
    //persisting
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{2});
    Thread.sleep(100);
    inputHandler.send(new Object[]{2});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("DeviceTempStream", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("TempStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{1});
    Thread.sleep(10);
    inputHandler.send(new Object[]{1});
    Thread.sleep(10);
    inputHandler.send(new Object[]{2});
    Thread.sleep(10);
    inputHandler.send(new Object[]{2});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count == 11);
    AssertJUnit.assertEquals(new Long(4), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test(dependsOnMethods = "persistenceTest10")
public void persistenceTest11() throws InterruptedException {
    log.info("persistence test 11 - batch window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume long );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.lengthBatch(2) " +
            "select *" +
            "insert all events into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timeStamp, inEvents, removeEvents);
            eventArrived = true;
            atomicCount.incrementAndGet();
            for (Event inEvent : inEvents) {
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
            }

            if (removeEvents != null) {
                for (Event removeEvent : removeEvents) {
                    lastValue = (Long) removeEvent.getData(2);
                }
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100L});
    inputHandler.send(new Object[]{"WSO2", 75.6f, 101L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 102L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 103L});
    SiddhiTestHelper.waitForEvents(100, 2, atomicCount, 10000);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(101), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(500);

    inputHandler.send(new Object[]{"WSO2", 75.6f, 50L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 50L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 50L});

    //restarting execution plan
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100L});

    //shutdown siddhi app
    Thread.sleep(500);
    SiddhiTestHelper.waitForEvents(100, 3, atomicCount, 10000);
    AssertJUnit.assertEquals(new Long(103), lastValue);
    siddhiAppRuntime.shutdown();
}
 

@Test(dependsOnMethods = "persistenceTest9")
public void persistenceTest10() throws InterruptedException {
    log.info("persistence test 10 - sort window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1') " +
            "from StockStream#window.sort(2,volume) " +
            "select volume " +
            "insert all events into outputStream ;";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timeStamp, inEvents, removeEvents);
            eventArrived = true;
            atomicCount.incrementAndGet();
            for (Event inEvent : inEvents) {
                count++;
            }

            if (removeEvents != null) {
                for (Event removeEvent : removeEvents) {
                    lastValueRemoved = (Integer) removeEvent.getData(0);
                }
            }
        }
    };
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    inputHandler.send(new Object[]{"WSO2", 55.6f, 100});
    inputHandler.send(new Object[]{"IBM", 75.6f, 300});
    inputHandler.send(new Object[]{"WSO2", 57.6f, 200});

    Thread.sleep(1000);
    AssertJUnit.assertEquals(3, count);
    AssertJUnit.assertTrue(eventArrived);
    // persisting
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"WSO2", 55.6f, 20});
    inputHandler.send(new Object[]{"WSO2", 57.6f, 40});

    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"WSO2", 55.6f, 20});

    SiddhiTestHelper.waitForEvents(100, 6, atomicCount, 10000);
    AssertJUnit.assertEquals(true, eventArrived);
    AssertJUnit.assertEquals(200, lastValueRemoved);
    siddhiAppRuntime.shutdown();

}
 

@Test(dependsOnMethods = "persistenceTest8")
public void persistenceTest9() throws InterruptedException {
    log.info("persistence test 9 - batch window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume long );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.timeBatch(300) " +
            "select * " +
            "insert all events into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timeStamp, inEvents, removeEvents);
            eventArrived = true;
            if (inEvents != null) {
                for (Event inEvent : inEvents) {
                    atomicCount.incrementAndGet();
                    AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                            "WSO2".equals(inEvent.getData(0)));
                    lastValue = (Long) inEvent.getData(2);
                }
            }

        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100L});
    inputHandler.send(new Object[]{"WSO2", 75.6f, 101L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 102L});
    Thread.sleep(400);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 103L});
    inputHandler.send(new Object[]{"WSO2", 75.6f, 104L});
    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);

    //persisting
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 105L});
    inputHandler.send(new Object[]{"WSO2", 75.6f, 106L});
    Thread.sleep(50);
    //restarting execution plan
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 107L});
    inputHandler.send(new Object[]{"IBM", 75.6f, 108L});
    Thread.sleep(10);

    SiddhiTestHelper.waitForEvents(100, 7, atomicCount, 10000);
    AssertJUnit.assertEquals(7, atomicCount.get());

    //shutdown siddhi app
    siddhiAppRuntime.shutdown();
}
 

@Test(dependsOnMethods = "persistenceTest7")
public void persistenceTest8() throws InterruptedException {
    log.info("persistence test 8 - window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.length(10) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timeStamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(200), lastValue);

    //persisting
    Thread.sleep(500);
    byte[] snapshot = siddhiAppRuntime.snapshot();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restore(snapshot);
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count == 6);
    AssertJUnit.assertEquals(new Long(400), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test(dependsOnMethods = "persistenceTest6")
public void persistenceTest7() throws InterruptedException {
    log.info("persistence test 7 - external time window with group by query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream (symbol string, price float, volume int, timestamp long);" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream#window.externalTime(timestamp,3 sec) " +
            "select symbol, price, sum(volume) as totalVol, timestamp " +
            "group by symbol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                if (count == 5) {
                    AssertJUnit.assertEquals(300L, inEvent.getData(2));
                }
                if (count == 6) {
                    AssertJUnit.assertEquals(100L, inEvent.getData(2));
                }
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    long currentTime = 0;

    inputHandler.send(new Object[]{"IBM", 75.1f, 100, currentTime + 1000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.2f, 100, currentTime + 2000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"IBM", 75.3f, 100, currentTime + 3000});

    Thread.sleep(500);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(3, count);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.4f, 100, currentTime + 4000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"IBM", 75.5f, 100, currentTime + 5000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100, currentTime + 6000});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(count, 6);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test(dependsOnMethods = "persistenceTest5")
public void persistenceTest6() throws InterruptedException {
    log.info("persistence test 6 - batch window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.timeBatch(10) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(500);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(2, count);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        log.error(e.getMessage(), e);
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(count, 6);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test(dependsOnMethods = "persistenceTest4")
public void persistenceTest5() throws InterruptedException {
    log.info("persistence test 5 - window restart expired event ");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.time(10 sec) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert all events into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            if (inEvents != null) {
                for (Event inEvent : inEvents) {
                    count++;
                    AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                    firstValue = (Long) inEvent.getData(2);
                }
            }
            if (removeEvents != null) {
                for (Event removeEvent : removeEvents) {
                    count++;
                    lastValue = (Long) removeEvent.getData(2);
                }
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(firstValue, 200);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    //shutdown siddhi app
    Thread.sleep(15000);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(400, firstValue);
    AssertJUnit.assertEquals(null, lastValue);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test
public void basicKafkaTestUsingBinaryMessage() throws InterruptedException, CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-0", "IntermediateTopic-0"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp1') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-0', bootstrap.servers='localhost:9092', " +
            "partition.no='0', sequence.id='ExternalDataRelayApp', is.binary.message='true', "
            + "@map(type='binary')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-0', group.id='test1', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +

            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-0', group.id='test1', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', is.binary.message='true', enable.auto.commit='false', @map(type='binary')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            LOG.info(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-0"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 5 events to the external topic.");

    // waits till the checkpointing task is done
    while (!externalDataRelayApp.persist().getFuture().isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit((Runnable) () -> KafkaTestUtil.kafkaPublisher(
            new String[]{"ExternalTopic-0"}, 1, 5, 100, false, null, true));
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(500);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "persistenceTest1")
public void persistenceTest2() throws InterruptedException {
    log.info("persistence test 2 - pattern count query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream Stream1 (symbol string, price float, volume int); " +
            "define stream Stream2 (symbol string, price float, volume int); " +
            "" +
            "@info(name = 'query1') " +
            "from e1=Stream1[price>20] <2:5> -> e2=Stream2[price>20] " +
            "select e1[0].price as price1_0, e1[1].price as price1_1, e1[2].price as price1_2, " +
            "   e1[3].price as price1_3, e2.price as price2 " +
            "insert into OutputStream ;";


    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertArrayEquals(new Object[]{25.6f, 47.6f, null, null, 45.7f}, inEvent.getData());
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler stream1 = siddhiAppRuntime.getInputHandler("Stream1");
    InputHandler stream2;
    siddhiAppRuntime.start();

    stream1.send(new Object[]{"WSO2", 25.6f, 100});
    Thread.sleep(100);
    stream1.send(new Object[]{"GOOG", 47.6f, 100});
    Thread.sleep(100);
    stream1.send(new Object[]{"GOOG", 13.7f, 100});
    Thread.sleep(100);

    AssertJUnit.assertEquals("Number of success events", 0, count);
    AssertJUnit.assertEquals("Event arrived", false, eventArrived);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    stream1 = siddhiAppRuntime.getInputHandler("Stream1");
    stream2 = siddhiAppRuntime.getInputHandler("Stream2");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    stream2.send(new Object[]{"IBM", 45.7f, 100});
    Thread.sleep(500);
    stream1.send(new Object[]{"GOOG", 47.8f, 100});
    Thread.sleep(500);
    stream2.send(new Object[]{"IBM", 55.7f, 100});
    Thread.sleep(500);

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals("Number of success events", 1, count);
    AssertJUnit.assertEquals("Event arrived", true, eventArrived);

}
 

@Test
public void persistenceTest1() throws InterruptedException {
    log.info("persistence test 1 - window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.length(10) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(200), lastValue);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed", e);
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= 6);
    AssertJUnit.assertEquals(new Long(400), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test(description = "Check if events are persisted when using delay window")
public void delayWindowTest7() throws InterruptedException {
    log.info("DelayWindow Test7: Testing persistence ");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String cseEventStream = "" +
            "define window delayWindow(symbol string, volume int) delay(1450);" +
            "define stream inputStream(symbol string, volume int);" +
            "define window timeWindow(symbol string) time(2 sec);";

    String query = "" +
            "@info(name='query1') " +
            "from inputStream " +
            "select symbol, volume " +
            "insert into delayWindow; " +
            "" +
            "@info(name = 'query2') " +
            "from delayWindow " +
            "select symbol, sum(volume) as totalVolume " +
            "insert into analyzeStream; " +
            "" +
            "@info(name='query3') " +
            "from inputStream " +
            "select symbol " +
            "insert into timeWindow; " +
            "" +
            "@info(name='query4') " +
            "from analyzeStream join timeWindow " +
            "on analyzeStream.symbol == timeWindow.symbol " +
            "select analyzeStream.symbol, analyzeStream.totalVolume " +
            "insert into outputStream; ";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(cseEventStream + query);

    siddhiAppRuntime.addCallback("outputStream", new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);
            count.addAndGet(events.length);
            for (Event event : events) {
                AssertJUnit.assertTrue(("IBM".equals(event.getData(0)) || "WSO2".equals(event.getData(0))));
            }
            lastValue = (Long) events[0].getData(1);
        }
    });

    InputHandler input = siddhiAppRuntime.getInputHandler("inputStream");
    siddhiAppRuntime.start();

    input.send(new Object[]{"IBM", 700});
    input.send(new Object[]{"WSO2", 750});

    SiddhiTestHelper.waitForEvents(100, 2, count, 4000);

    siddhiAppRuntime.persist();
    siddhiAppRuntime.shutdown();

    input = siddhiAppRuntime.getInputHandler("inputStream");
    siddhiAppRuntime.start();

    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    input.send(new Object[]{"WSO2", 600});
    SiddhiTestHelper.waitForEvents(100, 3, count, 4000);

    AssertJUnit.assertEquals(Long.valueOf(2050), lastValue);
    siddhiAppRuntime.shutdown();
}
 

@Test(description = "Check if events are persist when using session window")
public void testSessionWindow18() throws InterruptedException {
    log.info("SessionWindow Test18: Testing persistence ");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String purchaseEventStream = ""
            + "define stream purchaseEventStream (user string, item_number int, price float, quantity int); ";

    String query = ""
            + "@info(name = 'query0') "
            + "from purchaseEventStream#window.session(2 sec, user) "
            + "select * "
            + "insert all events into outputStream ;";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(purchaseEventStream + query);

    siddhiAppRuntime.addCallback("outputStream", new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            count.addAndGet(events.length);
            for (Event event : events) {
                innerAssertionsPassed = false;
                AssertJUnit.assertTrue(("101".equals(event.getData(1).toString()) ||
                        "102".equals(event.getData(1).toString())) ||
                        "103".equals(event.getData(1).toString()));
                innerAssertionsPassed = true;
            }

        }
    });

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("purchaseEventStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"user0", 101, 34.4, 5});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"user0", 102, 24.5, 2});

    siddhiAppRuntime.persist();
    siddhiAppRuntime.shutdown();

    inputHandler = siddhiAppRuntime.getInputHandler("purchaseEventStream");
    siddhiAppRuntime.start();

    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    inputHandler.send(new Object[]{"user0", 103, 24.5, 2});

    SiddhiTestHelper.waitForEvents(100, 3, count, 4200);
    AssertJUnit.assertTrue(innerAssertionsPassed);

    siddhiAppRuntime.shutdown();
}
 

@Test(dependsOnMethods = "testWithMultipleSeqIdsSinglePartitionWithTopicWiseThreading")
public void testWhenSequenceNumberIsNotAdded() throws InterruptedException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-6"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp6') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-6', bootstrap.servers='localhost:9092', " +
            "partition.no='0', @map(type='xml')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-6', group.id='test6', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +

            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp6') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-6', group.id='test6', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-6"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 10 events to the external topic.");

    Assert.assertEquals(count, 0);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "testWithMultiplePartitionsSingleTopic")
public void testWithMultiplePartitionsSingleTopicWithPartitionWiseThreading() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    KafkaTestUtil.createTopic(new String[]{"ExternalTopic-3"}, 1);
    KafkaTestUtil.createTopic(new String[]{"IntermediateTopic-3"}, 3);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp3') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-3', bootstrap.servers='localhost:9092', " +
            "partition.no='{{count}}', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "Define stream BarStream (symbol string, count long, total long); " +

            "@source(type='kafka', topic.list='ExternalTopic-3', group.id='test3', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count()%3 as count, count() as total insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp3') " +
            "Define stream BarStream1 (symbol string, count long, total long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-3', group.id='test3', " +
            "threading.option='partition.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0,1,2', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long, total long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-3"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 10 events to the external topic.");

    // Start the apps after events are published to the topic to let the events mixed up in the receiving order
    // for each partition
    LOG.info("Starting the Siddhi Apps");
    externalDataRelayApp.start();
    dataReceiveApp.start();

    Future perisistor = externalDataRelayApp.persist().getFuture();
    // waits till the checkpointing task is done
    while (!perisistor.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-3"}, 1, 5, 100, false, null, true);
        }
    });
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart so that
    // after the restart it will replay the last published 5 messages as it's not being persisted.
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(1000);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(new String[]{"ExternalTopic-3", "IntermediateTopic-3"});
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicTest")
public void testWithMultiplePartitionsSingleTopic() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    KafkaTestUtil.createTopic(new String[]{"ExternalTopic-2"}, 1);
    KafkaTestUtil.createTopic(new String[]{"IntermediateTopic-2"}, 3);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp2') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-2', bootstrap.servers='localhost:9092', " +
            "partition.no='{{count}}', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "Define stream BarStream (symbol string, count long, total long); " +

            "@source(type='kafka', topic.list='ExternalTopic-2', group.id='test2', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count()%3 as count, count() as total insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "Define stream BarStream1 (symbol string, count long, total long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-2', group.id='test2', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0,1,2', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long, total long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-2"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 10 events to the external topic.");

    // Start the apps after events are published to the topic to let the events mixed up in the receiving order
    // for each partition
    LOG.info("Starting the Siddhi Apps");
    externalDataRelayApp.start();
    dataReceiveApp.start();

    Future perisistor = externalDataRelayApp.persist().getFuture();
    // waits till the checkpointing task is done
    while (!perisistor.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-2"}, 1, 5, 100, false, null, true);
        }
    });
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart so that
    // after the restart it will replay the last published 5 messages as it's not being persisted.
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(1000);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(new String[]{"ExternalTopic-2", "IntermediateTopic-2"});
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicKafkaTestUsingBinaryMessageWithXmlMapper")
public void basicTest() throws InterruptedException, CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-1", "IntermediateTopic-1"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp1') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-1', bootstrap.servers='localhost:9092', " +
            "partition.no='0', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-1', group.id='test1', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-1', group.id='test1', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-1"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 5 events to the external topic.");

    // waits till the checkpointing task is done
    while (!externalDataRelayApp.persist().getFuture().isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit((Runnable) () -> KafkaTestUtil.kafkaPublisher(
            new String[]{"ExternalTopic-1"}, 1, 5, 100, false, null, true));
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(500);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicKafkaTestUsingBinaryMessage")
public void basicKafkaTestUsingBinaryMessageWithXmlMapper() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-xml", "IntermediateTopic-xml"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp1') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-xml', bootstrap.servers='localhost:9092', " +
            "partition.no='0', sequence.id='ExternalDataRelayApp', is.binary.message='true', "
            + "@map(type='xml')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-xml', group.id='test1', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +

            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-xml', group.id='test1', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', is.binary.message='true', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            LOG.info(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-xml"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 5 events to the external topic.");

    // waits till the checkpointing task is done
    while (!externalDataRelayApp.persist().getFuture().isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit((Runnable) () -> KafkaTestUtil.kafkaPublisher(
            new String[]{"ExternalTopic-xml"}, 1, 5, 100, false, null, true));
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(500);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}