Java Code Examples for org.apache.flink.streaming.api.functions.async.ResultFuture#complete()

@Test
public void testPop() {
	StreamElementQueue<Integer> queue = createStreamElementQueue(2);

	// add two elements to reach capacity
	putSuccessfully(queue, new Watermark(0L));
	ResultFuture<Integer> recordResult = putSuccessfully(queue, new StreamRecord<>(42, 1L));

	assertEquals(2, queue.size());

	// remove completed elements (watermarks are always completed)
	assertEquals(Arrays.asList(new Watermark(0L)), popCompleted(queue));
	assertEquals(1, queue.size());

	// now complete the stream record
	recordResult.complete(Collections.singleton(43));

	assertEquals(Arrays.asList(new StreamRecord<>(43, 1L)), popCompleted(queue));
	assertEquals(0, queue.size());
	assertTrue(queue.isEmpty());
}
 

/**
 * Tests that a put operation fails if the queue is full.
 */
@Test
public void testPutOnFull() throws Exception {
	final StreamElementQueue<Integer> queue = createStreamElementQueue(1);

	// fill up queue
	ResultFuture<Integer> resultFuture = putSuccessfully(queue, new StreamRecord<>(42, 0L));
	assertEquals(1, queue.size());

	// cannot add more
	putUnsuccessfully(queue, new StreamRecord<>(43, 1L));

	// popping the completed element frees the queue again
	resultFuture.complete(Collections.singleton(42 * 42));
	assertEquals(Arrays.asList(new StreamRecord<Integer>(42 * 42, 0L)), popCompleted(queue));

	// now the put operation should complete
	putSuccessfully(queue, new StreamRecord<>(43, 1L));
}
 

@Override
public void asyncInvoke(MetricEvent metricEvent, ResultFuture<MetricEvent> resultFuture) throws Exception {
    ps.setString(1, metricEvent.getName());
    ResultSet resultSet = ps.executeQuery();
    Map<String, Object> fields = metricEvent.getFields();
    if (resultSet.next()) {
        String thresholds = resultSet.getString("thresholds");
        String measurement = resultSet.getString("measurement");
        if (fields.get(measurement) != null && (double) fields.get(measurement) > Double.valueOf(thresholds)) {
            List<MetricEvent> list = new ArrayList<>();
            list.add(metricEvent);
            resultFuture.complete(Collections.singletonList(metricEvent));
        }
    }
}
 

public <I> void  complete(Collection<I> result){
    if (future instanceof ResultFuture){
        ResultFuture<I> resultFuture = (ResultFuture) future;
        resultFuture.complete(result);
    }else{
        ((CompletableFuture<Collection<I>>)future).complete(result);
    }
}
 

@Override
public void asyncInvoke(MetricEvent metricEvent, ResultFuture<MetricEvent> resultFuture) throws Exception {
    ps.setString(1, metricEvent.getName());
    ResultSet resultSet = ps.executeQuery();
    Map<String, Object> fields = metricEvent.getFields();
    if (resultSet.next()) {
        String thresholds = resultSet.getString("thresholds");
        String measurement = resultSet.getString("measurement");
        if (fields.get(measurement) != null && (double) fields.get(measurement) > Double.valueOf(thresholds)) {
            List<MetricEvent> list = new ArrayList<>();
            list.add(metricEvent);
            resultFuture.complete(Collections.singletonList(metricEvent));
        }
    }
}
 

/**
 * Tests that only the head element is pulled from the ordered queue if it has been
 * completed.
 */
@Test
public void testCompletionOrder() {
	final OrderedStreamElementQueue<Integer> queue = new OrderedStreamElementQueue<>(4);

	ResultFuture<Integer> entry1 = putSuccessfully(queue, new StreamRecord<>(1, 0L));
	ResultFuture<Integer> entry2 = putSuccessfully(queue, new StreamRecord<>(2, 1L));
	putSuccessfully(queue, new Watermark(2L));
	ResultFuture<Integer> entry4 = putSuccessfully(queue, new StreamRecord<>(3, 3L));

	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
	Assert.assertEquals(4, queue.size());
	Assert.assertFalse(queue.isEmpty());

	entry2.complete(Collections.singleton(11));
	entry4.complete(Collections.singleton(13));

	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
	Assert.assertEquals(4, queue.size());
	Assert.assertFalse(queue.isEmpty());

	entry1.complete(Collections.singleton(10));

	List<StreamElement> expected = Arrays.asList(
		new StreamRecord<>(10, 0L),
		new StreamRecord<>(11, 1L),
		new Watermark(2L),
		new StreamRecord<>(13, 3L));
	Assert.assertEquals(expected, popCompleted(queue));
	Assert.assertEquals(0, queue.size());
	Assert.assertTrue(queue.isEmpty());
}
 

@Override
public void timeout(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
	resultFuture.complete(Collections.singletonList(input * 3));
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	final Object lock = new Object();

	final long timeout = 100000L;
	final long timestamp = 1L;

	Environment environment = createMockEnvironment();

	ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);

	ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
	when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
	doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	Output<StreamRecord<Integer>> output = mock(Output.class);

	AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new AsyncFunction<Integer, Integer>() {
			private static final long serialVersionUID = -3718276118074877073L;

			@Override
			public void asyncInvoke(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
				resultFuture.complete(Collections.singletonList(input));
			}
		},
		timeout,
		1,
		AsyncDataStream.OutputMode.UNORDERED);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);

	synchronized (lock) {
		// processing an element will register a timeout
		operator.processElement(streamRecord);
	}

	synchronized (lock) {
		// closing the operator waits until all inputs have been processed
		operator.close();
	}

	// check that we actually outputted the result of the single input
	verify(output).collect(eq(streamRecord));
	verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));

	// check that we have cancelled our registered timeout
	verify(scheduledFuture).cancel(eq(true));
}
 

@Override
public void timeout(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
	resultFuture.complete(Collections.singletonList(input * 3));
}
 

/**
 * FLINK-5652
 * Tests that registered timers are properly canceled upon completion of a
 * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold
 * a reference on the StreamRecordQueueEntry.
 */
@Test
public void testTimeoutCleanup() throws Exception {
	final Object lock = new Object();

	final long timeout = 100000L;
	final long timestamp = 1L;

	Environment environment = createMockEnvironment();

	ScheduledFuture<?> scheduledFuture = mock(ScheduledFuture.class);

	ProcessingTimeService processingTimeService = mock(ProcessingTimeService.class);
	when(processingTimeService.getCurrentProcessingTime()).thenReturn(timestamp);
	doReturn(scheduledFuture).when(processingTimeService).registerTimer(anyLong(), any(ProcessingTimeCallback.class));

	StreamTask<?, ?> containingTask = mock(StreamTask.class);
	when(containingTask.getEnvironment()).thenReturn(environment);
	when(containingTask.getCheckpointLock()).thenReturn(lock);
	when(containingTask.getProcessingTimeService()).thenReturn(processingTimeService);

	StreamConfig streamConfig = new MockStreamConfig();
	streamConfig.setTypeSerializerIn1(IntSerializer.INSTANCE);

	Output<StreamRecord<Integer>> output = mock(Output.class);

	AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(
		new AsyncFunction<Integer, Integer>() {
			private static final long serialVersionUID = -3718276118074877073L;

			@Override
			public void asyncInvoke(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
				resultFuture.complete(Collections.singletonList(input));
			}
		},
		timeout,
		1,
		AsyncDataStream.OutputMode.UNORDERED);

	operator.setup(
		containingTask,
		streamConfig,
		output);

	operator.open();

	final StreamRecord<Integer> streamRecord = new StreamRecord<>(42, timestamp);

	synchronized (lock) {
		// processing an element will register a timeout
		operator.processElement(streamRecord);
	}

	synchronized (lock) {
		// closing the operator waits until all inputs have been processed
		operator.close();
	}

	// check that we actually outputted the result of the single input
	verify(output).collect(eq(streamRecord));
	verify(processingTimeService).registerTimer(eq(processingTimeService.getCurrentProcessingTime() + timeout), any(ProcessingTimeCallback.class));

	// check that we have cancelled our registered timeout
	verify(scheduledFuture).cancel(eq(true));
}
 

@Override
public void asyncInvoke(FetchUrl url, ResultFuture<FetchResultUrl> future)
        throws Exception {
    record(this.getClass(), url);

    final String domainKey = url.getUrlWithoutPath();
    Long nextFetchTime = _nextFetch.get(domainKey);
    long currentTime = System.currentTimeMillis();
    if ((nextFetchTime != null) && (currentTime < nextFetchTime)) {
        LOGGER.debug("Skipping (crawl-delay) " + url);
        future.complete(skipUrl(url, nextFetchTime));
        return;
    } else {
        _nextFetch.put(domainKey, currentTime + url.getCrawlDelay());
    }

    LOGGER.debug("Queueing for fetch: " + url);
    _executor.execute(new Runnable() {

        @Override
        public void run() {
            LOGGER.debug("Fetching " + url);

            try {
                FetchedResult result = _fetcher.get(url.getUrl(), null);
                FetchStatus fetchStatus = (result.getStatusCode() == HttpStatus.SC_OK) ?
                            FetchStatus.FETCHED
                        :   ExceptionUtils
                                .mapHttpStatusToFetchStatus(result.getStatusCode());
                FetchResultUrl fetchedUrl = new FetchResultUrl(url, fetchStatus, result.getFetchTime(),
                        result.getFetchedUrl(), result.getHeaders(), result.getContent(),
                        result.getContentType(), result.getResponseRate());

                _fetchCounts.increment();

                // If we got an error, put null in for fetchedUrl so we don't try to process it downstream.
                if (result.getStatusCode() != HttpStatus.SC_OK) {
                    String msg = String.format("Failed to fetch '%s' (%d)",
                            result.getFetchedUrl(), result.getStatusCode());
                    if (result.getStatusCode() == HttpStatus.SC_NOT_FOUND) {
                        LOGGER.trace(msg);
                    } else {
                        LOGGER.debug(msg);
                    }

                    // TODO set next fetch time to something valid, based on the error
                    LOGGER.trace("Forwarded failed URL to update status: '{}'",
                            result.getFetchedUrl());
                } else {
                    // TODO set next fetch time to something valid.
                    if (LOGGER.isTraceEnabled()) {
                        LOGGER.trace("Fetched {} bytes from '{}'",
                                result.getContentLength(), result.getFetchedUrl());
                        
                        LOGGER.trace("Forwarded fetched URL to be parsed: '{}'",
                                result.getFetchedUrl());
                    }
                }
                
                future.complete(Collections.singleton(fetchedUrl));
            } catch (Exception e) {
                LOGGER.trace("Failed to fetch '{}' due to {}", url, e.getMessage());

                if (e instanceof BaseFetchException) {
                    future.complete(Collections.singleton(new FetchResultUrl(url, ExceptionUtils.mapExceptionToFetchStatus(e),
                            System.currentTimeMillis())));
                    LOGGER.trace("Forwarded exception URL to update status: '{}'", url);
                } else {
                    throw new RuntimeException("Exception fetching " + url, e);
                }
            } catch (Throwable t) {
                LOGGER.error(String.format("Serious error trying to fetch '%s' due to %s", url,
                        t.getMessage()), t);
                throw new RuntimeException(t);
            }
        }
    });
}
 

/**
 * Tests that only elements before the oldest watermark are returned if they are completed.
 */
@Test
public void testCompletionOrder() {
	final UnorderedStreamElementQueue<Integer> queue = new UnorderedStreamElementQueue<>(8);

	ResultFuture<Integer> record1 = putSuccessfully(queue, new StreamRecord<>(1, 0L));
	ResultFuture<Integer> record2 = putSuccessfully(queue, new StreamRecord<>(2, 1L));
	putSuccessfully(queue, new Watermark(2L));
	ResultFuture<Integer> record3 = putSuccessfully(queue, new StreamRecord<>(3, 3L));
	ResultFuture<Integer> record4 = putSuccessfully(queue, new StreamRecord<>(4, 4L));
	putSuccessfully(queue, new Watermark(5L));
	ResultFuture<Integer> record5 = putSuccessfully(queue, new StreamRecord<>(5, 6L));
	ResultFuture<Integer> record6 = putSuccessfully(queue, new StreamRecord<>(6, 7L));

	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
	Assert.assertEquals(8, queue.size());
	Assert.assertFalse(queue.isEmpty());

	// this should not make any item completed, because R3 is behind W1
	record3.complete(Arrays.asList(13));

	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
	Assert.assertEquals(8, queue.size());
	Assert.assertFalse(queue.isEmpty());

	record2.complete(Arrays.asList(12));

	Assert.assertEquals(Arrays.asList(new StreamRecord<>(12, 1L)), popCompleted(queue));
	Assert.assertEquals(7, queue.size());
	Assert.assertFalse(queue.isEmpty());

	// Should not be completed because R1 has not been completed yet
	record6.complete(Arrays.asList(16));
	record4.complete(Arrays.asList(14));

	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
	Assert.assertEquals(7, queue.size());
	Assert.assertFalse(queue.isEmpty());

	// Now W1, R3, R4 and W2 are completed and should be pollable
	record1.complete(Arrays.asList(11));

	Assert.assertEquals(Arrays.asList(
			new StreamRecord<>(11, 0L),
		new Watermark(2L),
		new StreamRecord<>(13, 3L),
		new StreamRecord<>(14, 4L),
		new Watermark(5L),
		new StreamRecord<>(16, 7L)),
		popCompleted(queue));
	Assert.assertEquals(1, queue.size());
	Assert.assertFalse(queue.isEmpty());

	// only R5 left in the queue
	record5.complete(Arrays.asList(15));

	Assert.assertEquals(Arrays.asList(new StreamRecord<>(15, 6L)), popCompleted(queue));
	Assert.assertEquals(0, queue.size());
	Assert.assertTrue(queue.isEmpty());
	Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
}
 

@Override
public void timeout(Integer input, ResultFuture<Integer> resultFuture) throws Exception {
	resultFuture.complete(Collections.singletonList(input * 3));
}