Java Code Examples for io.siddhi.core.SiddhiAppRuntime#restoreLastRevision()

@Test
public void incrementalPersistenceTest4() throws InterruptedException {
    log.info("Incremental persistence test 4 - time batch window query");
    final int inputEventCount = 10;
    final int eventWindowSizeInSeconds = 2;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest4') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.timeBatch(" + eventWindowSizeInSeconds + " sec) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
                log.info("last value: " + lastValue);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 75.6f + i, 100});
    }
    Thread.sleep(4000);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(1000), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{"WSO2", 100.4f, 150});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 200.4f, 110});

    inputHandler.send(new Object[]{"IBM", 300.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 400.4f, 300});

    inputHandler.send(new Object[]{"IBM", 500.6f, 120});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 600.6f, 400});

    Thread.sleep(100);
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    siddhiAppRuntime.start();

    Thread.sleep(50);
    inputHandler.send(new Object[]{"IBM", 700.6f, 230});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 800.6f, 125});

    inputHandler.send(new Object[]{"IBM", 900.6f, 370});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 1000.6f, 140});

    //shutdown Siddhi app
    Thread.sleep(5000);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= (inputEventCount + 10));
    AssertJUnit.assertEquals(new Long(2045), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test(dependsOnMethods = "persistenceTest1")
public void persistenceTest2() throws InterruptedException {
    log.info("persistence test 2 - pattern count query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream Stream1 (symbol string, price float, volume int); " +
            "define stream Stream2 (symbol string, price float, volume int); " +
            "" +
            "@info(name = 'query1') " +
            "from e1=Stream1[price>20] <2:5> -> e2=Stream2[price>20] " +
            "select e1[0].price as price1_0, e1[1].price as price1_1, e1[2].price as price1_2, " +
            "   e1[3].price as price1_3, e2.price as price2 " +
            "insert into OutputStream ;";


    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertArrayEquals(new Object[]{25.6f, 47.6f, null, null, 45.7f}, inEvent.getData());
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler stream1 = siddhiAppRuntime.getInputHandler("Stream1");
    InputHandler stream2;
    siddhiAppRuntime.start();

    stream1.send(new Object[]{"WSO2", 25.6f, 100});
    Thread.sleep(100);
    stream1.send(new Object[]{"GOOG", 47.6f, 100});
    Thread.sleep(100);
    stream1.send(new Object[]{"GOOG", 13.7f, 100});
    Thread.sleep(100);

    AssertJUnit.assertEquals("Number of success events", 0, count);
    AssertJUnit.assertEquals("Event arrived", false, eventArrived);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    stream1 = siddhiAppRuntime.getInputHandler("Stream1");
    stream2 = siddhiAppRuntime.getInputHandler("Stream2");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    stream2.send(new Object[]{"IBM", 45.7f, 100});
    Thread.sleep(500);
    stream1.send(new Object[]{"GOOG", 47.8f, 100});
    Thread.sleep(500);
    stream2.send(new Object[]{"IBM", 55.7f, 100});
    Thread.sleep(500);

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals("Number of success events", 1, count);
    AssertJUnit.assertEquals("Event arrived", true, eventArrived);

}
 

@Test
public void persistenceTest1() throws InterruptedException {
    log.info("persistence test 1 - window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.length(10) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(200), lastValue);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed", e);
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= 6);
    AssertJUnit.assertEquals(new Long(400), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test
public void incrementalPersistenceTest12() throws InterruptedException {
    log.info("Incremental file persistence test 12 - length window query with max attribute aggregator");
    final int eventWindowSize = 5;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest12') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream#window.length(" + eventWindowSize + ") " +
            "select symbol, price, max(volume) as maxVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
                lastValue = new Long((Integer) inEvent.getData(2));
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 500});
    inputHandler.send(new Object[]{"IBM", 75.6f, 200});
    inputHandler.send(new Object[]{"IBM", 75.6f, 300});
    inputHandler.send(new Object[]{"IBM", 75.6f, 250});
    inputHandler.send(new Object[]{"IBM", 75.6f, 150});

    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(500L), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    //persisting for the second time to store the inc-snapshot
    siddhiAppRuntime.persist();
    Thread.sleep(100);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        log.error(e.getMessage(), e);
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    siddhiAppRuntime.start();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{"IBM", 100.4f, 280});
    AssertJUnit.assertEquals((Long) 300L, lastValue);

    inputHandler.send(new Object[]{"WSO2", 200.4f, 150});
    AssertJUnit.assertEquals((Long) 300L, lastValue);

    inputHandler.send(new Object[]{"IBM", 300.4f, 200});
    AssertJUnit.assertEquals((Long) 280L, lastValue);

    inputHandler.send(new Object[]{"WSO2", 400.4f, 270});
    AssertJUnit.assertEquals((Long) 280L, lastValue);

    inputHandler.send(new Object[]{"WSO2", 400.4f, 280});
    AssertJUnit.assertEquals((Long) 280L, lastValue);

    //shutdown Siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test
public void incrementalPersistenceTest11() throws InterruptedException {
    log.info("Incremental persistence test 11");
    AtomicInteger count = new AtomicInteger();
    AtomicLong ibmCount = new AtomicLong();
    AtomicLong wso2Count = new AtomicLong();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "@app:name('incrementalPersistenceTest11') " +
            "define stream StockQuote (symbol string, price float, volume int);" +
            "partition with (symbol of StockQuote) " +
            "begin " +
            "@info(name = 'query1') " +
            "from StockQuote#window.length(4)  " +
            "select symbol, count(price) as price " +
            "group by symbol insert into " +
            "OutStockStream ;  end ";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);

    StreamCallback streamCallback = new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);
            AssertJUnit.assertTrue("IBM".equals(events[0].getData(0)) || "WSO2".equals(events[0].getData(0)));
            count.addAndGet(events.length);

            for (Event event : events) {
                if ("IBM".equals(event.getData(0))) {
                    ibmCount.set((Long) event.getData(1));
                } else {
                    wso2Count.set((Long) event.getData(1));
                }
            }

            eventArrived = true;
        }
    };
    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockQuote");
    siddhiAppRuntime.start();

    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"IBM", 700f, 100}));
    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"WSO2", 60f, 50}));
    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"WSO2", 50f, 60}));
    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"WSO2", 40f, 60}));
    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"WSO2", 30f, 60}));

    siddhiAppRuntime.persist();

    Thread.sleep(2000);

    siddhiAppRuntime.shutdown();

    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    inputHandler = siddhiAppRuntime.getInputHandler("StockQuote");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    siddhiAppRuntime.start();
    Thread.sleep(2000);

    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"IBM", 800f, 100}));
    inputHandler.send(new Event(System.currentTimeMillis(), new Object[]{"WSO2", 20f, 60}));

    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(7, count.get());
    AssertJUnit.assertEquals(2, ibmCount.get());
    AssertJUnit.assertEquals(4, wso2Count.get());
    siddhiAppRuntime.shutdown();
}
 

@Test
public void incrementalPersistenceTest10() throws InterruptedException {
    log.info("Incremental persistence test 10 - partitioned sum with group-by on length windows.");
    final int inputEventCount = 10;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "@app:name('incrementalPersistenceTest10') "
            + "define stream cseEventStreamOne (symbol string, price float,volume int);"
            + "partition with (price>=100 as 'large' or price<100 as 'small' of cseEventStreamOne) " +
            "begin @info(name " +
            "= 'query1') from cseEventStreamOne#window.length(4) select symbol,sum(price) as price " +
            "group by symbol insert into " +
            "OutStockStream ;  end ";


    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    StreamCallback streamCallback = new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);

            eventArrived = true;
            if (events != null) {
                for (Event event : events) {
                    count++;
                    inEventsList.add(event.getData());
                    inEventCount.incrementAndGet();
                    lastValue = ((Double) event.getData(1)).longValue();
                }
            }
        }
    };

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 95f + i, 100});
        Thread.sleep(500);
        siddhiAppRuntime.persist();
    }

    siddhiAppRuntime.persist();
    inputHandler.send(new Object[]{"IBM", 205f, 100});
    Thread.sleep(2000);

    siddhiAppRuntime.shutdown();

    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);

    //siddhiAppRuntime.addCallback("query1", queryCallback);
    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed", e);
    }
    siddhiAppRuntime.start();
    Thread.sleep(2000);

    inputHandler.send(new Object[]{"IBM", 105f, 100});

    Thread.sleep(1000);

    AssertJUnit.assertEquals(new Long(414), lastValue);

    siddhiAppRuntime.shutdown();
}
 

@Test
public void incrementalPersistenceTest9() throws InterruptedException {
    log.info("Incremental persistence test 9 - min-max counting with group-by.");
    final int inputEventCount = 10;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String streams = "" +
            "@app:name('incrementalPersistenceTest9') " +
            "define stream TempStream (roomNo long, temp long); " +
            "define stream MaxTempStream (roomNo long, maxTemp long); ";

    String query = "" +
            "@info(name = 'query1') " +
            "from TempStream#window.length(10) " +
            "select roomNo, max(temp) as maxTemp " +
            "group by roomNo " +
            "insert into MaxTempStream; ";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            if (inEvents != null) {
                for (Event event : inEvents) {
                    count++;
                    inEventsList.add(event.getData());
                    inEventCount.incrementAndGet();
                    lastValue = (Long) event.getData(1);
                }
                eventArrived = true;
            }

            if (removeEvents != null) {
                removeEventCount = removeEventCount + removeEvents.length;
            }
        }
    };

    try {
        siddhiAppRuntime.addCallback("query1", queryCallback);
        InputHandler stockStream = siddhiAppRuntime.getInputHandler("TempStream");

        siddhiAppRuntime.start();
        for (int i = 0; i < inputEventCount; i++) {
            stockStream.send(new Object[]{i, 55L + i});
        }

        //persisting
        siddhiAppRuntime.persist();
        Thread.sleep(5000);

        siddhiAppRuntime.shutdown();

        siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);

        siddhiAppRuntime.addCallback("query1", queryCallback);

        stockStream = siddhiAppRuntime.getInputHandler("TempStream");
        //loading
        try {
            siddhiAppRuntime.restoreLastRevision();
        } catch (CannotRestoreSiddhiAppStateException e) {
            Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
        }
        siddhiAppRuntime.start();
        Thread.sleep(2000);

        stockStream.send(new Object[]{inputEventCount + 1, 1000L});

        //persisting
        siddhiAppRuntime.persist();
        Thread.sleep(2000);

        stockStream.send(new Object[]{inputEventCount + 2, 20L});

        siddhiAppRuntime.persist();
        Thread.sleep(2000);

        stockStream.send(new Object[]{4, 50L});

        //AssertJUnit.assertTrue(count <= (inputEventCount + 3));
        AssertJUnit.assertEquals(new Long(59), lastValue);
        AssertJUnit.assertEquals(true, eventArrived);


    } finally {
        siddhiAppRuntime.shutdown();
    }
}
 

@Test
public void incrementalPersistenceTest8() throws InterruptedException {
    log.info("Incremental persistence test 8 - min-max counting.");
    final int inputEventCount = 10;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String streams = "" +
            "@app:name('incrementalPersistenceTest8') " +
            "define stream TempStream (roomNo long, temp long); " +
            "define stream MaxTempStream (roomNo long, maxTemp long); ";

    String query = "" +
            "@info(name = 'query1') " +
            "from TempStream#window.length(10) " +
            "select roomNo, max(temp) as maxTemp " +
            "insert into MaxTempStream; ";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            if (inEvents != null) {
                for (Event event : inEvents) {
                    count++;
                    inEventsList.add(event.getData());
                    inEventCount.incrementAndGet();
                    lastValue = (Long) event.getData(1);
                }
                eventArrived = true;
            }

            if (removeEvents != null) {
                removeEventCount = removeEventCount + removeEvents.length;
            }
        }
    };

    try {
        siddhiAppRuntime.addCallback("query1", queryCallback);
        InputHandler stockStream = siddhiAppRuntime.getInputHandler("TempStream");

        siddhiAppRuntime.start();
        for (int i = 0; i < inputEventCount; i++) {
            stockStream.send(new Object[]{i, 55L + i});
        }

        //persisting
        siddhiAppRuntime.persist();
        Thread.sleep(5000);

        siddhiAppRuntime.shutdown();

        siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(streams + query);

        siddhiAppRuntime.addCallback("query1", queryCallback);

        stockStream = siddhiAppRuntime.getInputHandler("TempStream");
        //loading
        try {
            siddhiAppRuntime.restoreLastRevision();
        } catch (CannotRestoreSiddhiAppStateException e) {
            Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
        }
        siddhiAppRuntime.start();
        Thread.sleep(2000);

        stockStream.send(new Object[]{inputEventCount + 1, 1000L});

        //persisting
        siddhiAppRuntime.persist();
        Thread.sleep(2000);

        stockStream.send(new Object[]{inputEventCount + 2, 20L});

        siddhiAppRuntime.persist();
        Thread.sleep(2000);

        stockStream.send(new Object[]{inputEventCount + 3, 30L});


        AssertJUnit.assertTrue(count <= (inputEventCount + 3));
        AssertJUnit.assertEquals(new Long(1000), lastValue);
        AssertJUnit.assertEquals(true, eventArrived);


    } finally {
        siddhiAppRuntime.shutdown();
    }
}
 

@Test
public void incrementalPersistenceTest5() throws InterruptedException {
    log.info("Incremental persistence test 5 - external time window query");
    final int inputEventCount = 10;
    final int eventWindowSizeInSeconds = 2;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest5') " +
            "" +
            "define stream StockStream ( iij_timestamp long, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.externalTime(iij_timestamp, " + eventWindowSizeInSeconds + " sec) " +
            "select iij_timestamp, price, sum(volume) as totalVol " +
            "insert into OutStream";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                lastValue = (Long) inEvent.getData(2);
                log.info("last value: " + lastValue);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{(long) i, 75.6f + i, 100});
    }
    Thread.sleep(4000);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(1000), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{(1L + inputEventCount), 100.4f, 150});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{(2L + inputEventCount), 200.4f, 110});

    inputHandler.send(new Object[]{(3L + inputEventCount), 300.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{(4L + inputEventCount), 400.4f, 300});

    inputHandler.send(new Object[]{(5L + inputEventCount), 500.6f, 120});
    Thread.sleep(10);
    inputHandler.send(new Object[]{(6L + inputEventCount), 600.6f, 400});

    Thread.sleep(100);
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    siddhiAppRuntime.start();
    inputHandler.send(new Object[]{(7L + inputEventCount), 700.6f, 230});
    Thread.sleep(10);
    inputHandler.send(new Object[]{(8L + inputEventCount), 800.6f, 125});

    inputHandler.send(new Object[]{(9L + inputEventCount), 900.6f, 370});
    Thread.sleep(10);
    inputHandler.send(new Object[]{(10L + inputEventCount), 1000.6f, 140});

    //shutdown Siddhi app
    Thread.sleep(5000);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= (inputEventCount + 10));
    AssertJUnit.assertEquals(new Long(3045), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test(dependsOnMethods = "persistenceTest5")
public void persistenceTest6() throws InterruptedException {
    log.info("persistence test 6 - batch window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.timeBatch(10) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    Thread.sleep(500);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(2, count);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        log.error(e.getMessage(), e);
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.6f, 100});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(count, 6);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test(dependsOnMethods = "persistenceTest6")
public void persistenceTest7() throws InterruptedException {
    log.info("persistence test 7 - external time window with group by query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream (symbol string, price float, volume int, timestamp long);" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream#window.externalTime(timestamp,3 sec) " +
            "select symbol, price, sum(volume) as totalVol, timestamp " +
            "group by symbol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) || "WSO2".equals(inEvent.getData(0)));
                if (count == 5) {
                    AssertJUnit.assertEquals(300L, inEvent.getData(2));
                }
                if (count == 6) {
                    AssertJUnit.assertEquals(100L, inEvent.getData(2));
                }
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    long currentTime = 0;

    inputHandler.send(new Object[]{"IBM", 75.1f, 100, currentTime + 1000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.2f, 100, currentTime + 2000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"IBM", 75.3f, 100, currentTime + 3000});

    Thread.sleep(500);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(3, count);

    //persisting
    Thread.sleep(500);
    siddhiAppRuntime.persist();

    //restarting siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"IBM", 75.4f, 100, currentTime + 4000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"IBM", 75.5f, 100, currentTime + 5000});
    Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 75.6f, 100, currentTime + 6000});

    //shutdown siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertEquals(count, 6);
    AssertJUnit.assertEquals(true, eventArrived);

}
 

@Test
public void incrementalPersistenceTest3() throws InterruptedException {
    log.info("Incremental persistence test 3 - time window query");
    final int inputEventCount = 10;
    final int eventWindowSizeInSeconds = 2;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest3') " +
            "@app:playback " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.time(" + eventWindowSizeInSeconds + " sec) " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
                log.info("last value: " + lastValue);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(i, new Object[]{"IBM", 75.6f + i, 100});
    }
    Thread.sleep(4000);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(1000), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    inputHandler.send(3000, new Object[]{"WSO2", 100.4f, 150});
    //Thread.sleep(100);
    inputHandler.send(3010, new Object[]{"WSO2", 200.4f, 110});

    inputHandler.send(3020, new Object[]{"IBM", 300.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(3030, new Object[]{"WSO2", 400.4f, 300});

    inputHandler.send(3040, new Object[]{"IBM", 500.6f, 120});
    Thread.sleep(10);
    inputHandler.send(3050, new Object[]{"WSO2", 600.6f, 400});

    Thread.sleep(100);
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    try {
        //loading
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    siddhiAppRuntime.start();

    Thread.sleep(500);
    inputHandler.send(3500, new Object[]{"IBM", 700.6f, 230});
    Thread.sleep(10);
    inputHandler.send(3540, new Object[]{"WSO2", 800.6f, 125});

    inputHandler.send(3590, new Object[]{"IBM", 900.6f, 370});
    Thread.sleep(10);
    inputHandler.send(3600, new Object[]{"WSO2", 1000.6f, 140});

    //shutdown Siddhi app
    Thread.sleep(5000);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= (inputEventCount + 10));
    AssertJUnit.assertEquals(new Long(2045), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test
public void incrementalPersistenceTest2() throws InterruptedException {
    log.info("Incremental persistence test 2 - length batch window query");
    final int inputEventCount = 10;
    final int eventWindowSize = 4;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest2') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.lengthBatch(" + eventWindowSize + ") " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert all events into OutStream";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 75.6f + i, 100});
    }
    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(400), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{"WSO2", 100.4f, 150});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 200.4f, 110});

    inputHandler.send(new Object[]{"IBM", 300.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 400.4f, 300});

    inputHandler.send(new Object[]{"IBM", 500.6f, 120});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 600.6f, 400});

    Thread.sleep(100);
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed", e);
    }
    siddhiAppRuntime.start();

    Thread.sleep(500);
    inputHandler.send(new Object[]{"IBM", 700.6f, 230});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 800.6f, 125});

    inputHandler.send(new Object[]{"IBM", 900.6f, 370});
    Thread.sleep(10);
    inputHandler.send(new Object[]{"WSO2", 1000.6f, 140});

    //shutdown Siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= (inputEventCount + 6));
    AssertJUnit.assertEquals(new Long(865), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test
public void incrementalPersistenceTest1() throws InterruptedException {
    log.info("Incremental persistence test 1 - length window query");
    final int inputEventCount = 10;
    final int eventWindowSize = 4;

    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setIncrementalPersistenceStore(new IncrementalFileSystemPersistenceStore(storageFilePath));

    String siddhiApp = "" +
            "@app:name('incrementalPersistenceTest1') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1')" +
            "from StockStream[price>10]#window.length(" + eventWindowSize + ") " +
            "select symbol, price, sum(volume) as totalVol " +
            "insert into OutStream ";

    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timestamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timestamp, inEvents, removeEvents);
            eventArrived = true;
            for (Event inEvent : inEvents) {
                count++;
                AssertJUnit.assertTrue("IBM".equals(inEvent.getData(0)) ||
                        "WSO2".equals(inEvent.getData(0)));
                lastValue = (Long) inEvent.getData(2);
            }
        }
    };

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 75.6f + i, 100});
    }
    Thread.sleep(100);
    AssertJUnit.assertTrue(eventArrived);
    AssertJUnit.assertEquals(new Long(400), lastValue);

    //persisting
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{"IBM", 100.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 200.4f, 100});

    inputHandler.send(new Object[]{"IBM", 300.4f, 100});
    //Thread.sleep(100);
    inputHandler.send(new Object[]{"WSO2", 400.4f, 200});
    Thread.sleep(100);
    siddhiAppRuntime.persist();
    Thread.sleep(5000);

    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        log.error(e.getMessage(), e);
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }
    siddhiAppRuntime.start();
    Thread.sleep(5000);

    inputHandler.send(new Object[]{"IBM", 500.6f, 300});
    inputHandler.send(new Object[]{"WSO2", 600.6f, 400});

    //shutdown Siddhi app
    Thread.sleep(500);
    siddhiAppRuntime.shutdown();

    AssertJUnit.assertTrue(count <= (inputEventCount + 6));
    AssertJUnit.assertEquals(new Long(1000), lastValue);
    AssertJUnit.assertEquals(true, eventArrived);
}
 

@Test(dependsOnMethods = "persistenceTest9")
public void persistenceTest10() throws InterruptedException {
    log.info("persistence test 10 - sort window query");

    PersistenceStore persistenceStore = new InMemoryPersistenceStore();
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "" +
            "@app:name('Test') " +
            "" +
            "define stream StockStream ( symbol string, price float, volume int );" +
            "" +
            "@info(name = 'query1') " +
            "from StockStream#window.sort(2,volume) " +
            "select volume " +
            "insert all events into outputStream ;";

    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    QueryCallback queryCallback = new QueryCallback() {
        @Override
        public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
            EventPrinter.print(timeStamp, inEvents, removeEvents);
            eventArrived = true;
            atomicCount.incrementAndGet();
            for (Event inEvent : inEvents) {
                count++;
            }

            if (removeEvents != null) {
                for (Event removeEvent : removeEvents) {
                    lastValueRemoved = (Integer) removeEvent.getData(0);
                }
            }
        }
    };
    siddhiAppRuntime.addCallback("query1", queryCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    inputHandler.send(new Object[]{"WSO2", 55.6f, 100});
    inputHandler.send(new Object[]{"IBM", 75.6f, 300});
    inputHandler.send(new Object[]{"WSO2", 57.6f, 200});

    Thread.sleep(1000);
    AssertJUnit.assertEquals(3, count);
    AssertJUnit.assertTrue(eventArrived);
    // persisting
    siddhiAppRuntime.persist();

    inputHandler.send(new Object[]{"WSO2", 55.6f, 20});
    inputHandler.send(new Object[]{"WSO2", 57.6f, 40});

    Thread.sleep(500);
    siddhiAppRuntime.shutdown();
    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    siddhiAppRuntime.addCallback("query1", queryCallback);
    inputHandler = siddhiAppRuntime.getInputHandler("StockStream");
    siddhiAppRuntime.start();
    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    inputHandler.send(new Object[]{"WSO2", 55.6f, 20});

    SiddhiTestHelper.waitForEvents(100, 6, atomicCount, 10000);
    AssertJUnit.assertEquals(true, eventArrived);
    AssertJUnit.assertEquals(200, lastValueRemoved);
    siddhiAppRuntime.shutdown();

}
 

@Test(dependsOnMethods = "persistenceTest12")
public void persistenceTest13() throws InterruptedException {
    log.info("Persistence test 13 - partitioned sum with group-by on length windows.");
    final int inputEventCount = 10;
    PersistenceStore persistenceStore = new InMemoryPersistenceStore();
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(persistenceStore);

    String siddhiApp = "@app:name('incrementalPersistenceTest10') "
            + "define stream cseEventStreamOne (symbol string, price float,volume int);"
            + "partition with (price>=100 as 'large' or price<100 as 'small' of cseEventStreamOne) " +
            "begin " +
            "@info(name = 'query1') " +
            "from cseEventStreamOne#window.length(4) " +
            "select symbol,sum(price) as price " +
            "group by symbol " +
            "insert into OutStockStream;  " +
            "end ";


    SiddhiAppRuntime siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);
    StreamCallback streamCallback = new StreamCallback() {
        @Override
        public void receive(Event[] events) {
            EventPrinter.print(events);
            eventArrived = true;
            if (events != null) {
                for (Event event : events) {
                    count++;
                    lastValue = ((Double) event.getData(1)).longValue();
                }
            }
        }
    };

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    InputHandler inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    siddhiAppRuntime.start();

    for (int i = 0; i < inputEventCount; i++) {
        inputHandler.send(new Object[]{"IBM", 95f + i, 100});
        Thread.sleep(100);
        siddhiAppRuntime.persist();
    }

    inputHandler.send(new Object[]{"IBM", 205f, 100});
    Thread.sleep(100);

    siddhiAppRuntime.shutdown();

    siddhiAppRuntime = siddhiManager.createSiddhiAppRuntime(siddhiApp);

    siddhiAppRuntime.addCallback("OutStockStream", streamCallback);

    inputHandler = siddhiAppRuntime.getInputHandler("cseEventStreamOne");
    siddhiAppRuntime.start();

    Thread.sleep(1000);

    //loading
    try {
        siddhiAppRuntime.restoreLastRevision();
    } catch (CannotRestoreSiddhiAppStateException e) {
        Assert.fail("Restoring of Siddhi app " + siddhiAppRuntime.getName() + " failed");
    }

    Thread.sleep(1000);

    inputHandler.send(new Object[]{"IBM", 105f, 100});

    Thread.sleep(1000);

    AssertJUnit.assertEquals(new Long(414), lastValue);

    siddhiAppRuntime.shutdown();
}
 

@Test(dependsOnMethods = "testWithMultiplePartitionsSingleTopic")
public void testWithMultiplePartitionsSingleTopicWithPartitionWiseThreading() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    KafkaTestUtil.createTopic(new String[]{"ExternalTopic-3"}, 1);
    KafkaTestUtil.createTopic(new String[]{"IntermediateTopic-3"}, 3);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp3') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-3', bootstrap.servers='localhost:9092', " +
            "partition.no='{{count}}', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "Define stream BarStream (symbol string, count long, total long); " +

            "@source(type='kafka', topic.list='ExternalTopic-3', group.id='test3', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count()%3 as count, count() as total insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp3') " +
            "Define stream BarStream1 (symbol string, count long, total long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-3', group.id='test3', " +
            "threading.option='partition.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0,1,2', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long, total long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-3"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 10 events to the external topic.");

    // Start the apps after events are published to the topic to let the events mixed up in the receiving order
    // for each partition
    LOG.info("Starting the Siddhi Apps");
    externalDataRelayApp.start();
    dataReceiveApp.start();

    Future perisistor = externalDataRelayApp.persist().getFuture();
    // waits till the checkpointing task is done
    while (!perisistor.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-3"}, 1, 5, 100, false, null, true);
        }
    });
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart so that
    // after the restart it will replay the last published 5 messages as it's not being persisted.
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(1000);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(new String[]{"ExternalTopic-3", "IntermediateTopic-3"});
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicTest")
public void testWithMultiplePartitionsSingleTopic() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    KafkaTestUtil.createTopic(new String[]{"ExternalTopic-2"}, 1);
    KafkaTestUtil.createTopic(new String[]{"IntermediateTopic-2"}, 3);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp2') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-2', bootstrap.servers='localhost:9092', " +
            "partition.no='{{count}}', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "Define stream BarStream (symbol string, count long, total long); " +

            "@source(type='kafka', topic.list='ExternalTopic-2', group.id='test2', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count()%3 as count, count() as total insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "Define stream BarStream1 (symbol string, count long, total long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-2', group.id='test2', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0,1,2', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long, total long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-2"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 10 events to the external topic.");

    // Start the apps after events are published to the topic to let the events mixed up in the receiving order
    // for each partition
    LOG.info("Starting the Siddhi Apps");
    externalDataRelayApp.start();
    dataReceiveApp.start();

    Future perisistor = externalDataRelayApp.persist().getFuture();
    // waits till the checkpointing task is done
    while (!perisistor.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-2"}, 1, 5, 100, false, null, true);
        }
    });
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart so that
    // after the restart it will replay the last published 5 messages as it's not being persisted.
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(1000);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(new String[]{"ExternalTopic-2", "IntermediateTopic-2"});
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicKafkaTestUsingBinaryMessageWithXmlMapper")
public void basicTest() throws InterruptedException, CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-1", "IntermediateTopic-1"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp1') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-1', bootstrap.servers='localhost:9092', " +
            "partition.no='0', sequence.id='ExternalDataRelayApp', @map(type='xml')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-1', group.id='test1', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +
            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-1', group.id='test1', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            EventPrinter.print(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-1"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 5 events to the external topic.");

    // waits till the checkpointing task is done
    while (!externalDataRelayApp.persist().getFuture().isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit((Runnable) () -> KafkaTestUtil.kafkaPublisher(
            new String[]{"ExternalTopic-1"}, 1, 5, 100, false, null, true));
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(500);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}
 

@Test(dependsOnMethods = "basicKafkaTestUsingBinaryMessage")
public void basicKafkaTestUsingBinaryMessageWithXmlMapper() throws InterruptedException,
        CannotRestoreSiddhiAppStateException {
    LOG.info("Test to verify recovering process of a Siddhi node on a failure when Kafka is the event source");
    String topics[] = new String[]{"ExternalTopic-xml", "IntermediateTopic-xml"};
    KafkaTestUtil.createTopic(topics, 1);
    SiddhiManager siddhiManager = new SiddhiManager();
    siddhiManager.setPersistenceStore(new InMemoryPersistenceStore());

    final String externalDataRelayQuery = "@App:name('ExternalDataRelayApp1') " +
            "@info(name = 'ExternalEventRelayQuery') " +
            "@sink(type='kafka', topic='IntermediateTopic-xml', bootstrap.servers='localhost:9092', " +
            "partition.no='0', sequence.id='ExternalDataRelayApp', is.binary.message='true', "
            + "@map(type='xml')) " +
            "define stream BarStream (symbol string, count long); " +

            "@source(type='kafka', topic.list='ExternalTopic-xml', group.id='test1', " +
            "threading.option='topic.wise', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', @map(type='xml'))" +

            "Define stream FooStream (symbol string, price float, volume long); " +

            "from FooStream select symbol, count() as count insert into BarStream;";

    final String dataReceiveQuery = "@App:name('DataReceiveApp2') " +
            "define stream BarStream1 (symbol string, count long); " +

            "@info(name = 'DataReceiveQuery') " +
            "@source(type='kafka', topic.list='IntermediateTopic-xml', group.id='test1', " +
            "threading.option='topic.wise', seq.enabled='true', bootstrap.servers='localhost:9092', " +
            "partition.no.list='0', is.binary.message='true', @map(type='xml')) " +
            "Define stream FooStream1 (symbol string, count long);" +

            "from FooStream1 select * insert into BarStream1;";

    SiddhiAppRuntime externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    SiddhiAppRuntime dataReceiveApp = siddhiManager.createSiddhiAppRuntime(dataReceiveQuery);

    dataReceiveApp.addCallback("BarStream1", new StreamCallback() {
        @Override
        public synchronized void receive(Event[] events) {
            LOG.info(events);
            count += events.length;
        }
    });

    // Start the apps
    externalDataRelayApp.start();
    dataReceiveApp.start();

    // start publishing events to External Kafka topic
    Future eventSender = executorService.submit(new Runnable() {
        @Override
        public void run() {
            KafkaTestUtil.kafkaPublisher(
                    new String[]{"ExternalTopic-xml"}, 1, 10, 100, false, null, true);
        }
    });

    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished publishing 5 events to the external topic.");

    // waits till the checkpointing task is done
    while (!externalDataRelayApp.persist().getFuture().isDone()) {
        Thread.sleep(100);
    }
    LOG.info("Finished persisting the state of external event relay siddhi app");

    // Send more events after persisting the state
    eventSender = executorService.submit((Runnable) () -> KafkaTestUtil.kafkaPublisher(
            new String[]{"ExternalTopic-xml"}, 1, 5, 100, false, null, true));
    while (!eventSender.isDone()) {
        Thread.sleep(100);
    }

    // Shutting down the external relay app to mimic a node failure and starting it again like a restart
    LOG.info("Restarting the external relay Siddhi App to mimic a node failure and a restart");
    externalDataRelayApp.shutdown();
    externalDataRelayApp = siddhiManager.createSiddhiAppRuntime(externalDataRelayQuery);
    externalDataRelayApp.start();

    // Restore the state from last snapshot that was taken before shutdown
    externalDataRelayApp.restoreLastRevision();
    Thread.sleep(500);

    Assert.assertEquals(count, 15);

    KafkaTestUtil.deleteTopic(topics);
    externalDataRelayApp.shutdown();
    dataReceiveApp.shutdown();
    Thread.sleep(2000);
}