Java Code Examples for io.pravega.client.EventStreamClientFactory#createTransactionalEventWriter()

The following examples show how to use io.pravega.client.EventStreamClientFactory#createTransactionalEventWriter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlinkPravegaWriter.java    From flink-connectors with Apache License 2.0 5 votes vote down vote up 
AbstractInternalWriter(EventStreamClientFactory clientFactory, boolean txnWriter) {
    Serializer<T> eventSerializer = new FlinkSerializer<>(serializationSchema);
    EventWriterConfig writerConfig = EventWriterConfig.builder()
            .transactionTimeoutTime(txnLeaseRenewalPeriod)
            .build();
    watermark = Long.MIN_VALUE;
    if (txnWriter) {
        pravegaTxnWriter = clientFactory.createTransactionalEventWriter(writerId(), stream.getStreamName(), eventSerializer, writerConfig);
    } else {
        pravegaWriter = clientFactory.createEventWriter(writerId(), stream.getStreamName(), eventSerializer, writerConfig);
    }
}
Example 2
Source File: TurbineHeatSensor.java    From pravega-samples with Apache License 2.0 5 votes vote down vote up 
TransactionTemperatureSensors(TemperatureSensor sensor, int eventsPerSec, int secondsToRun, boolean
        isTransaction, EventStreamClientFactory factory) {
    super(sensor, eventsPerSec, secondsToRun, isTransaction, factory);
    EventWriterConfig eventWriterConfig =  EventWriterConfig.builder()
            .transactionTimeoutTime(DEFAULT_TXN_TIMEOUT_MS)
            .build();
    this.producerTxn = factory.createTransactionalEventWriter(streamName, SERIALIZER, eventWriterConfig);
    transaction = producerTxn.beginTxn();
}
Example 3
Source File: EndToEndStatsTest.java    From pravega with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 10000)
@SuppressWarnings("deprecation")
public void testStatsCount() throws Exception {
    StreamConfiguration config = StreamConfiguration.builder()
            .scalingPolicy(ScalingPolicy.fixed(1))
            .build();
    Controller controller = controllerWrapper.getController();
    controllerWrapper.getControllerService().createScope("test").get();
    controller.createStream("test", "test", config).get();
    @Cleanup
    EventStreamClientFactory clientFactory = new ClientFactoryImpl("test", controller);

    EventWriterConfig writerConfig = EventWriterConfig.builder().transactionTimeoutTime(10000).build();
    @Cleanup
    EventStreamWriter<String> eventWriter = clientFactory.createEventWriter("test", new JavaSerializer<>(),
            writerConfig);
    @Cleanup
    TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter("test", new JavaSerializer<>(),
            writerConfig);

    String[] tags = segmentTags(NameUtils.getQualifiedStreamSegmentName("test", "test", 0L));

    for (int i = 0; i < 10; i++) {
        eventWriter.writeEvent("test").get();
    }
    assertEventuallyEquals(10, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_EVENTS, tags).count()), 2000);
    assertEventuallyEquals(190, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_BYTES, tags).count()), 100);

    Transaction<String> transaction = txnWriter.beginTxn();
    for (int i = 0; i < 10; i++) {
        transaction.writeEvent("0", "txntest1");
    }
    assertEventuallyEquals(10, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_EVENTS, tags).count()), 2000);
    assertEventuallyEquals(190, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_BYTES, tags).count()), 100);

    transaction.commit();

    assertEventuallyEquals(20, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_EVENTS, tags).count()), 10000);
    assertEventuallyEquals(420, () -> (int) (statsRecorder.getRegistry().counter(SEGMENT_WRITE_BYTES, tags).count()), 100);
}
Example 4
Source File: StreamMetricsTest.java    From pravega with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 30000)
public void testTransactionMetrics() throws Exception {
    String txScopeName = "scopeTx";
    String txStreamName = "streamTx";

    controllerWrapper.getControllerService().createScope(txScopeName).get();
    if (!controller.createStream(txScopeName, txStreamName, config).get()) {
        log.error("Stream {} for tx testing already existed, exiting", txScopeName + "/" + txStreamName);
        return;
    }

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(txScopeName, ClientConfig.builder()
            .controllerURI(URI.create("tcp://localhost:" + controllerPort)).build());
    @Cleanup
    TransactionalEventStreamWriter<String> writer = clientFactory.createTransactionalEventWriter(Stream.of(txScopeName, txStreamName).getStreamName(),
            new JavaSerializer<>(), EventWriterConfig.builder().build());

    Transaction<String> transaction = writer.beginTxn();

    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.CREATE_TRANSACTION, streamTags(txScopeName, txStreamName)).count());

    transaction.writeEvent("Test");
    transaction.flush();
    transaction.commit();

    AssertExtensions.assertEventuallyEquals(true, () -> transaction.checkStatus().equals(Transaction.Status.COMMITTED), 10000);
    AssertExtensions.assertEventuallyEquals(true, () -> MetricRegistryUtils.getCounter(MetricsNames.COMMIT_TRANSACTION, streamTags(txScopeName, txStreamName)) != null, 10000);
    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.COMMIT_TRANSACTION, streamTags(txScopeName, txStreamName)).count());

    Transaction<String> transaction2 = writer.beginTxn();
    transaction2.writeEvent("Test");
    transaction2.abort();

    AssertExtensions.assertEventuallyEquals(true, () -> transaction2.checkStatus().equals(Transaction.Status.ABORTED), 10000);
    AssertExtensions.assertEventuallyEquals(true, () -> MetricRegistryUtils.getCounter(MetricsNames.ABORT_TRANSACTION, streamTags(txScopeName, txStreamName)) != null, 10000);
    assertEquals(1, (long) MetricRegistryUtils.getCounter(MetricsNames.ABORT_TRANSACTION, streamTags(txScopeName, txStreamName)).count());
}
Example 5
Source File: StreamRecreationTest.java    From pravega with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 40000)
@SuppressWarnings("deprecation")
public void testStreamRecreation() throws Exception {
    final String myScope = "myScope";
    final String myStream = "myStream";
    final String myReaderGroup = "myReaderGroup";
    final int numIterations = 10;

    // Create the scope and the stream.
    @Cleanup
    StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(myScope);
    @Cleanup
    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(myScope, controllerURI);
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
                                                                 .stream(Stream.of(myScope, myStream))
                                                                 .build();

    for (int i = 0; i < numIterations; i++) {
        log.info("Stream re-creation iteration {}.", i);
        final String eventContent = "myEvent" + String.valueOf(i);
        StreamConfiguration streamConfiguration = StreamConfiguration.builder()
                                                                     .scalingPolicy(ScalingPolicy.fixed(i + 1))
                                                                     .build();
        EventWriterConfig eventWriterConfig = EventWriterConfig.builder().build();
        streamManager.createStream(myScope, myStream, streamConfiguration);

        // Write a single event.
        @Cleanup
        EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(myScope, ClientConfig.builder().controllerURI(controllerURI).build());
        EventStreamWriter<String> writer = clientFactory.createEventWriter(myStream, new JavaSerializer<>(),
                                                                           eventWriterConfig);
        TransactionalEventStreamWriter<String> txnWriter = clientFactory.createTransactionalEventWriter(myStream,
                                                                                                        new JavaSerializer<>(),
                                                                                                        eventWriterConfig);

        // Write events regularly and with transactions.
        if (i % 2 == 0) {
            writer.writeEvent(eventContent).join();
        } else {
            Transaction<String> myTransaction = txnWriter.beginTxn();
            myTransaction.writeEvent(eventContent);
            myTransaction.commit();
            while (myTransaction.checkStatus() != Transaction.Status.COMMITTED) {
                Exceptions.handleInterrupted(() -> Thread.sleep(100));
            }
        }

        writer.close();

        // Read the event.
        readerGroupManager.createReaderGroup(myReaderGroup, readerGroupConfig);
        readerGroupManager.getReaderGroup(myReaderGroup).resetReaderGroup(readerGroupConfig);
        @Cleanup
        EventStreamReader<String> reader = clientFactory.createReader("myReader", myReaderGroup, new JavaSerializer<>(),
                ReaderConfig.builder().build());
        String readResult;
        do {
            readResult = reader.readNextEvent(1000).getEvent();
        } while (readResult == null);

        assertEquals("Wrong event read in re-created stream", eventContent, readResult);

        // Delete the stream.
        StreamInfo streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertFalse(streamInfo.isSealed());
        assertTrue("Unable to seal re-created stream.", streamManager.sealStream(myScope, myStream));
        streamInfo = streamManager.getStreamInfo(myScope, myStream);
        assertTrue(streamInfo.isSealed());
        assertTrue("Unable to delete re-created stream.", streamManager.deleteStream(myScope, myStream));
    }
}
Example 6
Source File: EndToEndTxnWithTest.java    From pravega with Apache License 2.0 4 votes vote down vote up 
private UUID createTxn(EventStreamClientFactory clientFactory, EventWriterConfig config, String streamName) {
    @Cleanup
    TransactionalEventStreamWriter<String> test = clientFactory.createTransactionalEventWriter("writer", streamName, new JavaSerializer<>(),
            config);
    return test.beginTxn().getTxnId();
}
Example 7
Source File: StreamMetricsTest.java    From pravega with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 30000)
public void testRollingTxnMetrics() throws Exception {
    String scaleRollingTxnScopeName = "scaleRollingTxnScope";
    String scaleRollingTxnStreamName = "scaleRollingTxnStream";

    controllerWrapper.getControllerService().createScope(scaleRollingTxnScopeName).get();
    if (!controller.createStream(scaleRollingTxnScopeName, scaleRollingTxnStreamName, config).get()) {
        fail("Stream " + scaleRollingTxnScopeName + "/" + scaleRollingTxnStreamName + " for scale testing already existed, test failed");
    }

    @Cleanup
    EventStreamClientFactory clientFactory = EventStreamClientFactory.withScope(scaleRollingTxnScopeName, ClientConfig.builder()
            .controllerURI(URI.create("tcp://localhost:" + controllerPort)).build());
    @Cleanup
    TransactionalEventStreamWriter<String> writer = clientFactory.createTransactionalEventWriter(Stream.of(scaleRollingTxnScopeName, scaleRollingTxnStreamName).getStreamName(),
            new JavaSerializer<>(), EventWriterConfig.builder().build());
    Transaction<String> transaction = writer.beginTxn();
    transaction.writeEvent("Transactional content");

    //split to 3 segments
    Map<Double, Double> keyRanges = new HashMap<>();
    keyRanges.put(0.0, 0.25);
    keyRanges.put(0.25, 0.75);
    keyRanges.put(0.75, 1.0);

    Stream scaleRollingTxnStream = new StreamImpl(scaleRollingTxnScopeName, scaleRollingTxnStreamName);
    if (!controller.scaleStream(scaleRollingTxnStream, Collections.singletonList(0L), keyRanges, executor).getFuture().get()) {
        fail("Scale stream: splitting segment into three failed, exiting");
    }

    assertEquals(3, (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_COUNT, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value());
    assertEquals(1, (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_SPLITS, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value());
    assertEquals(0, (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_MERGES, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value());

    transaction.flush();
    transaction.commit();

    String message = "Inconsistency found between metadata and metrics";
    AssertExtensions.assertEventuallyEquals(message, 3L, () -> (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_COUNT, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value(), 500, 30000);
    AssertExtensions.assertEventuallyEquals(message, 2L, () -> (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_SPLITS, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value(), 200, 30000);
    AssertExtensions.assertEventuallyEquals(message, 1L, () -> (long) MetricRegistryUtils.getGauge(MetricsNames.SEGMENTS_MERGES, streamTags(scaleRollingTxnScopeName, scaleRollingTxnStreamName)).value(), 200, 30000);
}
Example 8
Source File: AbstractReadWriteTest.java    From pravega with Apache License 2.0 4 votes vote down vote up 
private <T extends Serializable> TransactionalEventStreamWriter<T> instantiateTransactionalWriter(String writerId,
                                                                                                  EventStreamClientFactory clientFactory,
                                                                                                  String stream) {
    return clientFactory.createTransactionalEventWriter(writerId, stream, new JavaSerializer<T>(), buildWriterConfig());
}