Java Code Examples for org.apache.ignite.internal.IgniteEx#dataStreamer()

/** */
protected void loadData(IgniteEx node, int start, int end) {
    try (IgniteDataStreamer<Object, Object> streamer = node.dataStreamer(POI_CACHE_NAME)) {
        Random rnd = ThreadLocalRandom.current();

        for (int i = start; i < end; i++) {
            BinaryObject bo = node.binary().builder(POI_CLASS_NAME)
                .setField(NAME_FIELD_NAME, "POI_" + i, String.class)
                .setField(LATITUDE_FIELD_NAME, rnd.nextDouble(), Double.class)
                .setField(LONGITUDE_FIELD_NAME, rnd.nextDouble(), Double.class)
                .build();

            streamer.addData(i, bo);
        }
    }
}
 

/**
 * Reinit internal cache using passed ignite instance and fill it with data if required.
 *
 * @param ignite Node to get or create cache from.
 * @param fillData Whether the cache should be filled with new data or not.
 */
public void initCache(IgniteEx ignite, boolean fillData) {
    cache = ignite.getOrCreateCache(
        createCacheConfiguration()
            .setAtomicityMode(atomicityMode())
            .setCacheMode(cacheMode())
    );

    if (fillData) {
        try (IgniteDataStreamer<KeyType, ValueType> dataStreamer = ignite.dataStreamer(cache.getName())) {
            dataStreamer.allowOverwrite(true);

            for (int i = 0; i < entriesCount(); i++)
                dataStreamer.addData(createKey(i), createValue(i));
        }
    }
}
 

/** */
@Test
public void testCacheIdleVerifyNodeFilter() {
    IgniteEx ignite = crd;

    Object lastNodeCId = ignite.localNode().consistentId();

    ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
        .setAffinity(new RendezvousAffinityFunction(false, 32))
        .setNodeFilter(node -> !node.consistentId().equals(lastNodeCId))
        .setBackups(1));

    try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
        for (int i = 0; i < 100; i++)
            streamer.addData(i, i);
    }

    injectTestSystemOut();

    assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", DEFAULT_CACHE_NAME));

    assertContains(log, testOut.toString(), "no conflicts have been found");
}
 

/**
 * @throws Exception if failed.
 */
@Test
public void testDataStreamerModifiesReplicatedCacheSize() throws Exception {
    startGridsMultiThreaded(2);

    IgniteEx ignite = grid(0);

    ignite.createCache(
        new CacheConfiguration<>("test")
            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
            .setCacheMode(CacheMode.REPLICATED)
    );

    try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer("test")) {
        streamer.addData(1, "a");

        streamer.addData(keyInDifferentPartition(ignite, "test", 1), "b");
    }

    assertEquals(2, ignite.cache("test").size());

    assertEquals(1, grid(0).cache("test").localSize());
    assertEquals(1, grid(0).cache("test").localSize(BACKUP));

    assertEquals(1, grid(1).cache("test").localSize());
    assertEquals(1, grid(1).cache("test").localSize(BACKUP));
}
 

/**
 * Adding the "address" column and index for {@link Person} and
 * {@link Organization}, with new entries added for each of them.
 *
 * @param node Node.
 * @param cacheName Cache name.
 * @param addCnt How many entries add to table.
 * */
private void addColumnAndIdx(IgniteEx node, String cacheName, int addCnt) {
    IgniteCache<Object, Object> cache = node.cache(cacheName);

    cache.query(new SqlFieldsQuery("alter table Person add column orgAddr varchar")).getAll();
    cache.query(new SqlFieldsQuery("alter table Organization add column addr varchar")).getAll();

    cache.query(new SqlFieldsQuery("create index p_o_addr on Person (orgAddr)")).getAll();
    cache.query(new SqlFieldsQuery("create index o_addr on Organization (addr)")).getAll();

    int key = node.cachex(cacheName).size();

    try (IgniteDataStreamer<Object, Object> streamer = node.dataStreamer(cacheName)) {
        ThreadLocalRandom rand = ThreadLocalRandom.current();

        for (int i = 0; i < addCnt; i++) {
            streamer.addData(
                key++,
                new Person(rand.nextInt(), valueOf(rand.nextLong())).orgAddr(valueOf(rand.nextLong()))
            );

            streamer.addData(
                key++,
                new Organization(rand.nextInt(), valueOf(rand.nextLong())).addr(valueOf(rand.nextLong()))
            );
        }

        streamer.flush();
    }
}
 

/**
 * Loads data into a cache using stream data source.
 *
 * @param ignite Ignite.
 * @param cache Cache.
 * @param stream Key-value source stream.
 */
protected void load(IgniteEx ignite, String cache, Stream<Integer> stream) {
    try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cache)) {
        final Iterator<Integer> it = stream.iterator();

        while (it.hasNext()) {
            Integer key = it.next();

            s.addData(key, key);
        }
    }
}
 

/**
 * @param ig Ig.
 * @param cacheName Cache name.
 * @param startKey Start key range.
 * @param cnt Count.
 */
private void populateCache(IgniteEx ig, String cacheName, int startKey, int cnt) throws IgniteCheckedException {
    try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(cacheName)) {
        for (int i = startKey; i < startKey + cnt; i++)
            streamer.addData(i, new byte[5 * 1000]);
    }

    GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig.context().cache().context().database();

    dbMgr.waitForCheckpoint("test");
}
 

/**
 * @param ignite Ignite.
 * @param keysCnt Keys count.
 */
private void loadData(IgniteEx ignite, int keysCnt, int multiplier) {
    log.info("Load data: keys=" + keysCnt);

    try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
        streamer.allowOverwrite(true);

        for (int k = 0; k < keysCnt; k++)
            streamer.addData(k, k * multiplier);
    }
}
 

/**
 * Too many deletes in single transaction may overflow {@link GridDhtLocalPartition#rmvQueue} and entries will be
 * deleted synchronously in {@link GridDhtLocalPartition#onDeferredDelete(int, KeyCacheObject, GridCacheVersion)}.
 * This should not corrupt internal map state in {@link GridDhtLocalPartition}.
 *
 * @throws Exception If failed.
 */
public void checkTxLeavesObjectsInLocalPartition(CacheConfiguration<Integer, String> ccfg,
    TransactionConcurrency optimistic, TransactionIsolation isolation) throws Exception {
    IgniteEx igniteEx = grid(0);

    igniteEx.getOrCreateCache(ccfg);

    try (IgniteDataStreamer<Integer, String> dataStreamer = igniteEx.dataStreamer(DEFAULT_CACHE_NAME)) {
        for (int i = 0; i < CACHE_ENTRIES_COUNT; i++)
            dataStreamer.addData(i, UUID.randomUUID().toString());
    }

    IgniteEx client = startClientGrid(getConfiguration()
            .setIgniteInstanceName(UUID.randomUUID().toString()));

    awaitPartitionMapExchange();

    assertEquals(CACHE_ENTRIES_COUNT, client.getOrCreateCache(DEFAULT_CACHE_NAME).size());

    try (Transaction tx = client.transactions().txStart(optimistic, isolation)) {
        IgniteCache<Integer, String> cache = client.getOrCreateCache(cacheConfiguration());

        for (int v = 0; v < CACHE_ENTRIES_COUNT; v++) {
            cache.get(v);

            cache.remove(v);
        }

        tx.commit();
    }

    GridTestUtils.waitForCondition(
        () -> igniteEx.context().cache().cacheGroups().stream()
            .filter(CacheGroupContext::userCache)
            .flatMap(cgctx -> cgctx.topology().localPartitions().stream())
            .mapToInt(GridDhtLocalPartition::internalSize)
            .max().orElse(-1) == 0,
        500L
    );
}
 

/**
 * @param ignite Ignite.
 * @param cache Cache.
 * @param keys Keys.
 */
private void load(IgniteEx ignite, String cache, List<Integer> keys) {
    try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cache)) {
        for (Integer key : keys)
            s.addData(key, key);
    }
}
 

/**
 * @throws Exception If failed.
 */
@Test
public void testCacheIdleVerifyPrintLostPartitions() throws Exception {
    IgniteEx ignite = startGrids(3);

    ignite.cluster().active(true);

    ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
        .setAffinity(new RendezvousAffinityFunction(false, 16))
        .setCacheMode(PARTITIONED)
        .setPartitionLossPolicy(READ_ONLY_SAFE)
        .setBackups(1));

    try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
        for (int i = 0; i < 10000; i++)
            streamer.addData(i, new byte[i]);
    }

    String g1Name = grid(1).name();

    stopGrid(1);

    cleanPersistenceDir(g1Name);

    //Start node 2 with empty PDS. Rebalance will be started.
    startGrid(1);

    //During rebalance stop node 3. Rebalance will be stopped which lead to lost partitions.
    stopGrid(2);

    injectTestSystemOut();

    assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--yes"));

    assertContains(log, testOut.toString(), "LOST partitions:");
}
 

/**
 * Runs multithreaded put scenario (no data streamer). Load is generated to page store and to WAL.
 *
 * @throws Exception if failed.
 */
@Test
public void testDataStreamerContinuousPutMultithreaded() throws Exception {
    try {
        // System.setProperty(IgniteSystemProperties.IGNITE_DIRTY_PAGES_PARALLEL, "true");
        // System.setProperty(IgniteSystemProperties.IGNITE_DIRTY_PAGES_SORTED_STORAGE, "true");
        System.setProperty(IgniteSystemProperties.IGNITE_USE_ASYNC_FILE_IO_FACTORY, "false");
        System.setProperty(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED, "speed");
        System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "true");

        setWalArchAndWorkToSameVal = true;

        customWalMode = WALMode.BACKGROUND;

        final IgniteEx ignite = startGrid(1);

        ignite.active(true);

        final int threads = 1;
        Runtime.getRuntime().availableProcessors();

        final int recsPerThread = CONTINUOUS_PUT_RECS_CNT / threads;

        final ProgressWatchdog watchdog = new ProgressWatchdog(ignite, "put", PUT_THREAD);

        IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer(CACHE_NAME);

        streamer.perNodeBufferSize(12);

        final Collection<Callable<?>> tasks = new ArrayList<>();
        for (int j = 0; j < threads; j++) {
            final int finalJ = j;

            tasks.add((Callable<Void>)() -> {
                for (int i = finalJ * recsPerThread; i < ((finalJ + 1) * recsPerThread); i++)
                    streamer.addData(i, new HugeIndexedObject(i));

                return null;
            });
        }

        final IgniteCache<Object, HugeIndexedObject> cache = ignite.cache(CACHE_NAME);
        ScheduledExecutorService svcReport = Executors.newScheduledThreadPool(1);

        AtomicInteger size = new AtomicInteger();
        svcReport.scheduleAtFixedRate(
            () -> {
                int newSize = cache.size();
                int oldSize = size.getAndSet(newSize);

                watchdog.reportProgress(newSize - oldSize);
            },
            250, 250, TimeUnit.MILLISECONDS);

        watchdog.start();
        GridTestUtils.runMultiThreaded(tasks, PUT_THREAD);
        streamer.close();

        watchdog.stopping();
        stopGrid(1);

        watchdog.stop();

        ProgressWatchdog.stopPool(svcReport);

        if (VERIFY_STORAGE)
            runVerification(threads, recsPerThread);
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * @throws Exception if failed.
 */
@Test
public void testWalRecoveryWithDynamicCache() throws Exception {
    try {
        IgniteEx ignite = startGrid(1);

        ignite.active(true);

        CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();

        dynCacheCfg.setName("dyncache");
        dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
        dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
        dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
        dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
        dynCacheCfg.setReadFromBackup(true);

        ignite.getOrCreateCache(dynCacheCfg);

        try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
            for (int i = 0; i < KEYS_COUNT; ++i) {
                if (i % (KEYS_COUNT / 100) == 0)
                    info("Loading " + i * 100 / KEYS_COUNT + "%");

                dataLdr.addData(i, new IndexedObject(i));
            }
        }

        for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
            stopGrid(1, true);

            info("Restart #" + restartCnt);
            U.sleep(500);

            ignite = startGrid(1);

            ignite.active(true);

            ThreadLocalRandom locRandom = ThreadLocalRandom.current();

            IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);

            for (int i = 0; i < KEYS_COUNT; ++i)
                assertNotNull(cache.get(locRandom.nextInt(KEYS_COUNT)));
        }
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * @throws Exception if failed.
 */
@Test
public void testWalRecoveryWithDynamicCacheLargeObjects() throws Exception {
    try {
        IgniteEx ignite = startGrid(1);

        ignite.active(true);

        CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();

        dynCacheCfg.setName("dyncache");
        dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
        dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
        dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
        dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
        dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
        dynCacheCfg.setReadFromBackup(true);

        ignite.getOrCreateCache(dynCacheCfg);

        try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
            for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
                if (i % (LARGE_KEYS_COUNT / 100) == 0)
                    info("Loading " + i * 100 / LARGE_KEYS_COUNT + "%");

                IndexedObject obj = new IndexedObject(i);

                obj.payload = new byte[PAGE_SIZE + 2];

                dataLdr.addData(i, obj);
            }
        }

        for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
            stopGrid(1, true);

            info("Restart #" + restartCnt);
            U.sleep(500);

            ignite = startGrid(1);

            ignite.active(true);

            ThreadLocalRandom locRandom = ThreadLocalRandom.current();

            IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);

            for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
                IndexedObject val = cache.get(locRandom.nextInt(LARGE_KEYS_COUNT));

                assertNotNull(val);

                assertEquals(PAGE_SIZE + 2, val.payload.length);
            }
        }
    }
    finally {
        stopAllGrids();
    }
}
 

/**
 * Load data in range specified.
 *
 * @param ignite Ignite.
 * @param from Firs index loading data.
 * @param to Last index loading data.
 */
private void preloadData(IgniteEx ignite, int from, int to) {
    try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
        for (int i = from; i < to; i++)
            streamer.addData(i, "Val " + i);
    }
}