org.apache.flink.api.common.io.FinalizeOnMaster Java Examples

The following examples show how to use org.apache.flink.api.common.io.FinalizeOnMaster. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: InputOutputFormatVertex.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void finalizeOnMaster(ClassLoader loader) throws Exception {
	final InputOutputFormatContainer formatContainer = initInputOutputformatContainer(loader);

	final ClassLoader original = Thread.currentThread().getContextClassLoader();
	try {
		// set user classloader before calling user code
		Thread.currentThread().setContextClassLoader(loader);

		// configure input formats and invoke finalizeGlobal()
		Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
		for (Map.Entry<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> entry : outputFormats.entrySet()) {
			final OutputFormat<?> outputFormat;

			try {
				outputFormat = entry.getValue().getUserCodeObject();
				outputFormat.configure(formatContainer.getParameters(entry.getKey()));
			} catch (Throwable t) {
				throw new Exception("Configuring the output format (" + getFormatDescription(entry.getKey()) + ") failed: "
					+ t.getMessage(), t);
			}

			if (outputFormat instanceof FinalizeOnMaster) {
				((FinalizeOnMaster) outputFormat).finalizeGlobal(getParallelism());
			}
		}

	} finally {
		// restore original classloader
		Thread.currentThread().setContextClassLoader(original);
	}
}
Example #2
Source File: InputOutputFormatVertex.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void finalizeOnMaster(ClassLoader loader) throws Exception {
	final InputOutputFormatContainer formatContainer = initInputOutputformatContainer(loader);

	final ClassLoader original = Thread.currentThread().getContextClassLoader();
	try {
		// set user classloader before calling user code
		Thread.currentThread().setContextClassLoader(loader);

		// configure input formats and invoke finalizeGlobal()
		Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
		for (Map.Entry<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> entry : outputFormats.entrySet()) {
			final OutputFormat<?> outputFormat;

			try {
				outputFormat = entry.getValue().getUserCodeObject();
				outputFormat.configure(formatContainer.getParameters(entry.getKey()));
			} catch (Throwable t) {
				throw new Exception("Configuring the output format (" + getFormatDescription(entry.getKey()) + ") failed: "
					+ t.getMessage(), t);
			}

			if (outputFormat instanceof FinalizeOnMaster) {
				((FinalizeOnMaster) outputFormat).finalizeGlobal(getParallelism());
			}
		}

	} finally {
		// restore original classloader
		Thread.currentThread().setContextClassLoader(original);
	}
}
Example #3
Source File: GenericDataSinkBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
Example #4
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
Example #5
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}