org.apache.flink.api.common.io.RichOutputFormat Java Examples

The following examples show how to use org.apache.flink.api.common.io.RichOutputFormat. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TupleOutputFormatSinkFunction.java    From alibaba-flink-connectors with Apache License 2.0 6 votes vote down vote up 
@Override
public void open(Configuration config) throws IOException {
	if (RichOutputFormat.class.isAssignableFrom(outputFormat.getClass())) {
		((RichOutputFormat) outputFormat).setRuntimeContext(getRuntimeContext());
	}
	outputFormat.configure(config);
	outputFormat.open(
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
	if (outputFormat instanceof HasRetryTimeout) {
		retryTimeout = ((HasRetryTimeout) outputFormat).getRetryTimeout();
	}
	LOG.info(
		"Initialized OutputFormatSinkFunction of {}/{} task.",
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
}
Example #2
Source File: OutputFormatSinkFunction.java    From alibaba-flink-connectors with Apache License 2.0 6 votes vote down vote up 
@Override
public void open(Configuration config) throws IOException {
	if (RichOutputFormat.class.isAssignableFrom(outputFormat.getClass())) {
		((RichOutputFormat) outputFormat).setRuntimeContext(getRuntimeContext());
	}
	outputFormat.configure(config);
	outputFormat.open(
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
	if (outputFormat instanceof HasRetryTimeout) {
		retryTimeout = ((HasRetryTimeout) outputFormat).getRetryTimeout();
	}
	LOG.info(
		"Initialized OutputFormatSinkFunction of {}/{} task.",
		getRuntimeContext().getIndexOfThisSubtask(),
		getRuntimeContext().getNumberOfParallelSubtasks());
}
Example #3
Source File: OutputFormatSinkFunctionTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
Example #4
Source File: OutputFormatSinkFunction.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
Example #5
Source File: CollectionExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up 
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
Example #6
Source File: JdbcDB.java    From Alink with Apache License 2.0 5 votes vote down vote up 
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    TypeInformation[] types = schema.getFieldTypes();
    String[] colNames = schema.getFieldNames();
    int[] parameterTypes = new int[types.length];
    for (int i = 0; i < types.length; ++i) {
        parameterTypes[i] = JdbcTypeConverter.getIntegerSqlType(types[i]);
    }
    StringBuilder sbd = new StringBuilder();
    sbd.append("INSERT INTO ").append(tableName).append(" (").append(colNames[0]);
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append(colNames[i]);
    }
    sbd.append(") VALUES (?");
    for (int i = 1; i < colNames.length; i++) {
        sbd.append(",").append("?");
    }
    sbd.append(")");

    return JDBCOutputFormat.buildJDBCOutputFormat()
            .setUsername(userName)
            .setPassword(password)
            .setDBUrl(getDbUrl())
            .setQuery(sbd.toString())
            .setDrivername(getDriverName())
            .setBatchInterval(5000)
            .setSqlTypes(parameterTypes)
            .finish();
}
Example #7
Source File: OutputFormatSinkFunctionTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
Example #8
Source File: OutputFormatSinkFunction.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
Example #9
Source File: CollectionExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up 
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
Example #10
Source File: OutputFormatSinkFunctionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void setRuntimeContext() throws Exception {
	RuntimeContext mockRuntimeContext = Mockito.mock(RuntimeContext.class);

	// Make sure setRuntimeContext of the rich output format is called
	RichOutputFormat<?> mockRichOutputFormat = Mockito.mock(RichOutputFormat.class);
	new OutputFormatSinkFunction<>(mockRichOutputFormat).setRuntimeContext(mockRuntimeContext);
	Mockito.verify(mockRichOutputFormat, Mockito.times(1)).setRuntimeContext(Mockito.eq(mockRuntimeContext));

	// Make sure setRuntimeContext work well when output format is not RichOutputFormat
	OutputFormat<?> mockOutputFormat = Mockito.mock(OutputFormat.class);
	new OutputFormatSinkFunction<>(mockOutputFormat).setRuntimeContext(mockRuntimeContext);
}
Example #11
Source File: OutputFormatSinkFunction.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void setRuntimeContext(RuntimeContext context) {
	super.setRuntimeContext(context);
	if (format instanceof RichOutputFormat) {
		((RichOutputFormat) format).setRuntimeContext(context);
	}
}
Example #12
Source File: CollectionExecutor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private <IN> void executeDataSink(GenericDataSinkBase<?> sink, int superStep) throws Exception {
	Operator<?> inputOp = sink.getInput();
	if (inputOp == null) {
		throw new InvalidProgramException("The data sink " + sink.getName() + " has no input.");
	}
	
	@SuppressWarnings("unchecked")
	List<IN> input = (List<IN>) execute(inputOp);
	
	@SuppressWarnings("unchecked")
	GenericDataSinkBase<IN> typedSink = (GenericDataSinkBase<IN>) sink;

	// build the runtime context and compute broadcast variables, if necessary
	TaskInfo taskInfo = new TaskInfo(typedSink.getName(), 1, 0, 1, 0);
	RuntimeUDFContext ctx;

	MetricGroup metrics = new UnregisteredMetricsGroup();
		
	if (RichOutputFormat.class.isAssignableFrom(typedSink.getUserCodeWrapper().getUserCodeClass())) {
		ctx = superStep == 0 ? new RuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics) :
				new IterationRuntimeUDFContext(taskInfo, userCodeClassLoader, executionConfig, cachedFiles, accumulators, metrics);
	} else {
		ctx = null;
	}

	typedSink.executeOnCollections(input, ctx, executionConfig);
}
Example #13
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
Example #14
Source File: FakeDBBase.java    From Alink with Apache License 2.0 4 votes vote down vote up 
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    return null;
}
Example #15
Source File: HiveDB.java    From Alink with Apache License 2.0 4 votes vote down vote up 
@Override
public RichOutputFormat createFormat(String tableName, TableSchema schema) {
    throw new UnsupportedOperationException("not supported.");
}
Example #16
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
Example #17
Source File: GenericDataSinkBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
Example #18
Source File: BaseDB.java    From Alink with Apache License 2.0 2 votes vote down vote up 
/**
 * Create Flink OutputFormat from a table in db.
 *
 * @param tableName table name
 * @param schema    format schema.
 * @return Flink OutputFormat
 */
public abstract RichOutputFormat createFormat(String tableName, TableSchema schema);