Java Code Examples for org.apache.spark.api.java.JavaPairRDD#values()

@Test
public void testBasicStream() throws Exception {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);
    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(tenRows, 2).mapToPair(new FailsFunction(3));
    StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    srdd.submit();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        LOG.trace(execRow);
        count++;
        assertNotNull(execRow);
        assertTrue(execRow.getColumn(1).getInt() < 10);
    }
    assertEquals(10, count);
}
 

@Test
public void testFailureBoundary() throws Exception {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);
    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(tenRows, 20).mapToPair(new FailsFunction(4));
    StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    srdd.submit();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        LOG.trace(execRow);
        count++;
        assertNotNull(execRow);
        assertTrue(execRow.getColumn(1).getInt() < 10);
    }
    assertEquals(10, count);
}
 

@Test
public void testBasicStream() throws Exception {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);
    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(tenRows, 10);
    StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    srdd.submit();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        LOG.trace(execRow);
        count++;
        assertNotNull(execRow);
        assertTrue(execRow.getColumn(1).getInt() < 10);
    }
    assertEquals(10, count);
}
 

@Test
public void testOffset() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(-1, 60000);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = 60000;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(100000-60000, count);
}
 

@Test
public void testLimit() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(400, 0);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(400, count);
}
 

@Test
public void testOffsetLimit() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(400, 30000);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = 30000;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(400, count);
}
 

@Test
public void testSmallLimit() throws StandardException {
    int limit = 2000;
    int offset = 0;
    int total = 4000;
    int batches = 2;
    int batchSize = 512;
    StreamListener<ExecRow> sl = new StreamListener<>(limit, offset, batches, batchSize);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < total; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 1);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), null, sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort(), batches, batchSize);
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = offset;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(limit, count);
}
 

@Test
public void testSmallOffsetLimit() throws StandardException {
    int limit = 100;
    int offset = 2000;
    int total = 4000;
    StreamListener<ExecRow> sl = new StreamListener<>(limit, offset);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < total; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 1);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = offset;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(limit, count);
}
 

@Test
public void testBlockingLargeOddPartitions() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        count++;
        assertNotNull(execRow);
    }
    assertEquals(100000, count);
}
 

@Test
public void testBlockingLarge() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 12);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        count++;
        assertNotNull(execRow);
    }
    assertEquals(100000, count);
}
 

@Test
public void testBlocking() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 10000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 6);
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                LOG.error(e);
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        count++;
        assertNotNull(execRow);
    }
    assertEquals(10000, count);
}
 

@Test
public void testFailureAfterRecoveryWarmup() throws StandardException, FileNotFoundException, UnsupportedEncodingException {
    int size = 100000;
    int batches = 2;
    int batchSize = 512;
    StreamListener<ExecRow> sl = new StreamListener<>(-1, 0, batches, batchSize);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < size; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 2).sortByKey().mapToPair(new FailsTwiceFunction(10000, 2000));
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), null, sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort(), batches, batchSize);
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        count++;
    }
    assertEquals(size, count);
}
 

@Test
public void testFailureDuringRecoveryWarmup() throws StandardException, FileNotFoundException, UnsupportedEncodingException {
    int size = 100000;
    int batches = 2;
    int batchSize = 512;
    StreamListener<ExecRow> sl = new StreamListener<>(-1, 0, batches, batchSize);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < size; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 2).sortByKey().mapToPair(new FailsTwiceFunction(10000, 100));
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), null, sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort(), batches, batchSize);
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        count++;
    }
    assertEquals(size, count);
}
 

@Test
public void testFailureAfterLimit() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(40000, 300);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13).mapToPair(new FailsFunction(40301));;
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = 300;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(40000, count);
}
 

@Test
public void testFailureBeforeOffset() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(40000, 300);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13).mapToPair(new FailsFunction(200));;
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = 300;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(40000, count);
}
 

@Test
public void testFailureBeforeLargeOffset() throws StandardException {
    StreamListener<ExecRow> sl = new StreamListener<>(400, 30000);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 13).mapToPair(new FailsFunction(29500));;
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    int first = 30000;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        assertEquals(count+first, execRow.getColumn(1).getInt());
        count++;
    }
    assertEquals(400, count);
}
 

@Test
public void testBlockingLarge() throws StandardException, FileNotFoundException, UnsupportedEncodingException {
    int size = 100000;
    int batches = 2;
    int batchSize = 512;
    StreamListener<ExecRow> sl = new StreamListener<>(-1, 0, batches, batchSize);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < size; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 12).sortByKey().mapToPair(new FailsFunction(10000));
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), null, sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort(), batches, batchSize);
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        count++;
    }
    assertEquals(size, count);
}
 

@Test
public void testBlockingMedium() throws StandardException, FileNotFoundException, UnsupportedEncodingException {
    int size = 20000;
    int batches = 2;
    int batchSize = 512;
    StreamListener<ExecRow> sl = new StreamListener<>(-1, 0, batches, batchSize);
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < size; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 2).sortByKey().mapToPair(new FailsFunction(5000));
    final StreamableRDD srdd = new StreamableRDD(rdd.values(), null, sl.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort(), batches, batchSize);
    new Thread() {
        @Override
        public void run() {
            try {
                srdd.submit();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }.start();
    Iterator<ExecRow> it = sl.getIterator();
    int count = 0;
    while (it.hasNext()) {
        ExecRow execRow = it.next();
        assertNotNull(execRow);
        count++;
    }
    assertEquals(size, count);
}
 

@Test
public void testConcurrentQueries() throws StandardException, ExecutionException, InterruptedException {
    final StreamListener<ExecRow> sl1 = new StreamListener<>();
    final StreamListener<ExecRow> sl2 = new StreamListener<>();
    final StreamListener<ExecRow> sl3 = new StreamListener<>();
    HostAndPort hostAndPort = server.getHostAndPort();
    server.register(sl1);
    server.register(sl2);
    server.register(sl3);

    List<Tuple2<ExecRow,ExecRow>> manyRows = new ArrayList<>();
    for(int i = 0; i < 100000; ++i) {
        manyRows.add(new Tuple2<ExecRow, ExecRow>(getExecRow(i, 1), getExecRow(i, 2)));
    }

    JavaPairRDD<ExecRow, ExecRow> rdd = SpliceSpark.getContextUnsafe().parallelizePairs(manyRows, 12);
    final StreamableRDD srdd1 = new StreamableRDD(rdd.values(), sl1.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    final StreamableRDD srdd2 = new StreamableRDD(rdd.values().map(new Function<ExecRow,ExecRow>() {
        @Override
        public ExecRow call(ExecRow o) throws Exception {
            o.getColumn(1).setValue(0);
            return o;
        }
    }), sl2.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    final StreamableRDD srdd3 = new StreamableRDD(rdd.values(), sl3.getUuid(), hostAndPort.getHostText(), hostAndPort.getPort());
    for (final StreamableRDD srdd : Arrays.asList(srdd1, srdd2, srdd3)) {
        new Thread() {
            @Override
            public void run() {
                try {
                    srdd.submit();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }

            }
        }.start();
    }
    // We collect them asynchronously into memory so we are able to iterate over them at the same time. Otherwise
    // tasks for the third RDD might be blocked by tasks in other RDDs, and we are not consuming elements from the
    // other iterators so they can become unblocked.
    ExecutorService executor = Executors.newFixedThreadPool(3);
    Future<List<ExecRow>> future1 = executor.submit(new Callable<List<ExecRow>>() {
        @Override
        public List<ExecRow> call() throws Exception {
            return IteratorUtils.toList(sl1.getIterator());
        }
    });
    Future<List<ExecRow>> future2 = executor.submit(new Callable<List<ExecRow>>() {
        @Override
        public List<ExecRow> call() throws Exception {
            return IteratorUtils.toList(sl2.getIterator());
        }
    });
    Future<List<ExecRow>> future3 = executor.submit(new Callable<List<ExecRow>>() {
        @Override
        public List<ExecRow> call() throws Exception {
            return IteratorUtils.toList(sl3.getIterator());
        }
    });
    Iterator<ExecRow> it1 = future1.get().iterator();
    Iterator<ExecRow> it2 = future2.get().iterator();
    Iterator<ExecRow> it3 = future3.get().iterator();
    int count = 0;
    while (it1.hasNext()) {
        ExecRow r1 = it1.next();
        ExecRow r2 = it2.next();
        ExecRow r3 = it3.next();
        count++;
        assertNotNull(r1);
        assertNotNull(r2);
        assertNotNull(r3);
        assertEquals(0, r2.getColumn(1).getInt());
        assertEquals(r1.getColumn(1), r3.getColumn(1));
        assertEquals(r1.getColumn(2), r2.getColumn(2));
    }
    assertEquals(100000, count);
}
 

@Override
public void runUpdate(JavaSparkContext sparkContext,
                      long timestamp,
                      JavaPairRDD<Object,M> newKeyMessageData,
                      JavaPairRDD<Object,M> pastKeyMessageData,
                      String modelDirString,
                      TopicProducer<String,String> modelUpdateTopic)
    throws IOException, InterruptedException {

  Objects.requireNonNull(newKeyMessageData);

  JavaRDD<M> newData = newKeyMessageData.values();
  JavaRDD<M> pastData = pastKeyMessageData == null ? null : pastKeyMessageData.values();

  if (newData != null) {
    newData.cache();
    // This forces caching of the RDD. This shouldn't be necessary but we see some freezes
    // when many workers try to materialize the RDDs at once. Hence the workaround.
    newData.foreachPartition(p -> {});
  }
  if (pastData != null) {
    pastData.cache();
    pastData.foreachPartition(p -> {});
  }

  List<List<?>> hyperParameterCombos = HyperParams.chooseHyperParameterCombos(
      getHyperParameterValues(), hyperParamSearch, candidates);

  Path modelDir = new Path(modelDirString);
  Path tempModelPath = new Path(modelDir, ".temporary");
  Path candidatesPath = new Path(tempModelPath, Long.toString(System.currentTimeMillis()));

  FileSystem fs = FileSystem.get(modelDir.toUri(), sparkContext.hadoopConfiguration());
  fs.mkdirs(candidatesPath);

  Path bestCandidatePath = findBestCandidatePath(
      sparkContext, newData, pastData, hyperParameterCombos, candidatesPath);

  Path finalPath = new Path(modelDir, Long.toString(System.currentTimeMillis()));
  if (bestCandidatePath == null) {
    log.info("Unable to build any model");
  } else {
    // Move best model into place
    fs.rename(bestCandidatePath, finalPath);
  }
  // Then delete everything else
  fs.delete(candidatesPath, true);

  if (modelUpdateTopic == null) {
    log.info("No update topic configured, not publishing models to a topic");
  } else {
    // Push PMML model onto update topic, if it exists
    Path bestModelPath = new Path(finalPath, MODEL_FILE_NAME);
    if (fs.exists(bestModelPath)) {
      FileStatus bestModelPathFS = fs.getFileStatus(bestModelPath);
      PMML bestModel = null;
      boolean modelNeededForUpdates = canPublishAdditionalModelData();
      boolean modelNotTooLarge = bestModelPathFS.getLen() <= maxMessageSize;
      if (modelNeededForUpdates || modelNotTooLarge) {
        // Either the model is required for publishAdditionalModelData, or required because it's going to
        // be serialized to Kafka
        try (InputStream in = fs.open(bestModelPath)) {
          bestModel = PMMLUtils.read(in);
        }
      }

      if (modelNotTooLarge) {
        modelUpdateTopic.send("MODEL", PMMLUtils.toString(bestModel));
      } else {
        modelUpdateTopic.send("MODEL-REF", fs.makeQualified(bestModelPath).toString());
      }

      if (modelNeededForUpdates) {
        publishAdditionalModelData(
            sparkContext, bestModel, newData, pastData, finalPath, modelUpdateTopic);
      }
    }
  }

  if (newData != null) {
    newData.unpersist();
  }
  if (pastData != null) {
    pastData.unpersist();
  }
}