Java Code Examples for org.apache.flink.runtime.memory.MemoryManager#computeNumberOfPages()
The following examples show how to use
org.apache.flink.runtime.memory.MemoryManager#computeNumberOfPages() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HashJoinIteratorBase.java From flink with Apache License 2.0 | 6 votes |
|
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer,
TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer,
TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager,
IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBloomFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new MutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager,
useBloomFilters);
}
Example 2
| Source File: ReusingBuildSecondReOpenableHashJoinIterator.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager, IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBitmapFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager, useBitmapFilters);
}
Example 3
| Source File: ReusingBuildFirstReOpenableHashJoinIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager, IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBitmapFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager, useBitmapFilters);
}
Example 4
| Source File: NonReusingBuildFirstReOpenableHashJoinIterator.java From flink with Apache License 2.0 | 6 votes |
|
@Override
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer, TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer, TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager, IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBitmapFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new ReOpenableMutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager, useBitmapFilters);
}
Example 5
| Source File: HashJoinIteratorBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer,
TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer,
TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager,
IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBloomFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new MutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager,
useBloomFilters);
}
Example 6
| Source File: HashJoinIteratorBase.java From flink with Apache License 2.0 | 6 votes |
|
public <BT, PT> MutableHashTable<BT, PT> getHashJoin(
TypeSerializer<BT> buildSideSerializer,
TypeComparator<BT> buildSideComparator,
TypeSerializer<PT> probeSideSerializer,
TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> pairComparator,
MemoryManager memManager,
IOManager ioManager,
AbstractInvokable ownerTask,
double memoryFraction,
boolean useBloomFilters) throws MemoryAllocationException {
final int numPages = memManager.computeNumberOfPages(memoryFraction);
final List<MemorySegment> memorySegments = memManager.allocatePages(ownerTask, numPages);
return new MutableHashTable<BT, PT>(buildSideSerializer, probeSideSerializer,
buildSideComparator, probeSideComparator, pairComparator,
memorySegments, ioManager,
useBloomFilters);
}
Example 7
| Source File: ChainedReduceCombineDriver.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void openTask() throws Exception {
// open the stub first
final Configuration stubConfig = config.getStubParameters();
BatchTask.openUserCode(reducer, stubConfig);
// instantiate the serializer / comparator
serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer();
comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator();
MemoryManager memManager = parent.getEnvironment().getMemoryManager();
final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
memory = memManager.allocatePages(parent, numMemoryPages);
LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
switch (strategy) {
case SORTED_PARTIAL_REDUCE:
// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
if (comparator.supportsSerializationWithKeyNormalization() &&
serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
} else {
sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
}
break;
case HASHED_PARTIAL_REDUCE:
table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
table.open();
reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled);
break;
}
}
Example 8
| Source File: SynchronousChainedCombineDriver.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void openTask() throws Exception {
// open the stub first
final Configuration stubConfig = this.config.getStubParameters();
BatchTask.openUserCode(this.combiner, stubConfig);
// ----------------- Set up the sorter -------------------------
// instantiate the serializer / comparator
final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
this.serializer = serializerFactory.getSerializer();
TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
this.groupingComparator = groupingComparatorFactory.createComparator();
MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
this.memory = memManager.allocatePages(this.parent, numMemoryPages);
// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
if (sortingComparator.supportsSerializationWithKeyNormalization() &&
this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
{
this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
} else {
this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
}
if (LOG.isDebugEnabled()) {
LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
Example 9
| Source File: SynchronousChainedCombineDriver.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void openTask() throws Exception {
// open the stub first
final Configuration stubConfig = this.config.getStubParameters();
BatchTask.openUserCode(this.combiner, stubConfig);
// ----------------- Set up the sorter -------------------------
// instantiate the serializer / comparator
final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
this.serializer = serializerFactory.getSerializer();
TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
this.groupingComparator = groupingComparatorFactory.createComparator();
MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
this.memory = memManager.allocatePages(this.parent, numMemoryPages);
// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
if (sortingComparator.supportsSerializationWithKeyNormalization() &&
this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
{
this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
} else {
this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
}
if (LOG.isDebugEnabled()) {
LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
}
Example 10
| Source File: ChainedReduceCombineDriver.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void openTask() throws Exception {
// open the stub first
final Configuration stubConfig = config.getStubParameters();
BatchTask.openUserCode(reducer, stubConfig);
// instantiate the serializer / comparator
serializer = config.<T>getInputSerializer(0, userCodeClassLoader).getSerializer();
comparator = config.<T>getDriverComparator(0, userCodeClassLoader).createComparator();
MemoryManager memManager = parent.getEnvironment().getMemoryManager();
final int numMemoryPages = memManager.computeNumberOfPages(config.getRelativeMemoryDriver());
memory = memManager.allocatePages(parent, numMemoryPages);
LOG.debug("ChainedReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
switch (strategy) {
case SORTED_PARTIAL_REDUCE:
// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
if (comparator.supportsSerializationWithKeyNormalization() &&
serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
} else {
sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
}
break;
case HASHED_PARTIAL_REDUCE:
table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
table.open();
reduceFacade = table.new ReduceFacade(reducer, outputCollector, objectReuseEnabled);
break;
}
}
Example 11
| Source File: BatchTask.java From flink with Apache License 2.0 | 4 votes |
|
/**
*
* NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
* {@code #initInputSerializersAndComparators(int)}!
*/
protected void initLocalStrategies(int numInputs) throws Exception {
final MemoryManager memMan = getMemoryManager();
final IOManager ioMan = getIOManager();
this.localStrategies = new CloseableInputProvider<?>[numInputs];
this.inputs = new MutableObjectIterator<?>[numInputs];
this.excludeFromReset = new boolean[numInputs];
this.inputIsCached = new boolean[numInputs];
this.inputIsAsyncMaterialized = new boolean[numInputs];
this.materializationMemory = new int[numInputs];
// set up the local strategies first, such that the can work before any temp barrier is created
for (int i = 0; i < numInputs; i++) {
initInputLocalStrategy(i);
}
// we do another loop over the inputs, because we want to instantiate all
// sorters, etc before requesting the first input (as this call may block)
// we have two types of materialized inputs, and both are replayable (can act as a cache)
// The first variant materializes in a different thread and hence
// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
// the second variant spills to the side and will not read unless the result is also consumed
// in a pipelined fashion.
this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
this.tempBarriers = new TempBarrier<?>[numInputs];
for (int i = 0; i < numInputs; i++) {
final int memoryPages;
final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
final boolean cached = this.config.isInputCached(i);
this.inputIsAsyncMaterialized[i] = async;
this.inputIsCached[i] = cached;
if (async || cached) {
memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
if (memoryPages <= 0) {
throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
}
this.materializationMemory[i] = memoryPages;
} else {
memoryPages = 0;
}
if (async) {
@SuppressWarnings({ "unchecked", "rawtypes" })
TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
barrier.startReading();
this.tempBarriers[i] = barrier;
this.inputs[i] = null;
} else if (cached) {
@SuppressWarnings({ "unchecked", "rawtypes" })
SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
this.resettableInputs[i] = iter;
this.inputs[i] = iter;
}
}
}
Example 12
| Source File: LeftOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
Example 13
| Source File: LeftOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
Example 14
| Source File: RightOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case RIGHT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.RIGHT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case RIGHT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
case RIGHT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
default:
throw new Exception("Unsupported driver strategy for right outer join driver: " + driverStrategy.name());
}
}
Example 15
| Source File: LeftOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
Example 16
| Source File: FullOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case FULL_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.FULL,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case FULL_OUTER_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
case FULL_OUTER_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
default:
throw new Exception("Unsupported driver strategy for full outer join driver: " + driverStrategy.name());
}
}
Example 17
| Source File: RightOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case RIGHT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.RIGHT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case RIGHT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
case RIGHT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
default:
throw new Exception("Unsupported driver strategy for right outer join driver: " + driverStrategy.name());
}
}
Example 18
| Source File: BatchTask.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
/**
*
* NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
* {@code #initInputSerializersAndComparators(int)}!
*/
protected void initLocalStrategies(int numInputs) throws Exception {
final MemoryManager memMan = getMemoryManager();
final IOManager ioMan = getIOManager();
this.localStrategies = new CloseableInputProvider<?>[numInputs];
this.inputs = new MutableObjectIterator<?>[numInputs];
this.excludeFromReset = new boolean[numInputs];
this.inputIsCached = new boolean[numInputs];
this.inputIsAsyncMaterialized = new boolean[numInputs];
this.materializationMemory = new int[numInputs];
// set up the local strategies first, such that the can work before any temp barrier is created
for (int i = 0; i < numInputs; i++) {
initInputLocalStrategy(i);
}
// we do another loop over the inputs, because we want to instantiate all
// sorters, etc before requesting the first input (as this call may block)
// we have two types of materialized inputs, and both are replayable (can act as a cache)
// The first variant materializes in a different thread and hence
// acts as a pipeline breaker. this one should only be there, if a pipeline breaker is needed.
// the second variant spills to the side and will not read unless the result is also consumed
// in a pipelined fashion.
this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
this.tempBarriers = new TempBarrier<?>[numInputs];
for (int i = 0; i < numInputs; i++) {
final int memoryPages;
final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
final boolean cached = this.config.isInputCached(i);
this.inputIsAsyncMaterialized[i] = async;
this.inputIsCached[i] = cached;
if (async || cached) {
memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
if (memoryPages <= 0) {
throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
}
this.materializationMemory[i] = memoryPages;
} else {
memoryPages = 0;
}
if (async) {
@SuppressWarnings({ "unchecked", "rawtypes" })
TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages);
barrier.startReading();
this.tempBarriers[i] = barrier;
this.inputs[i] = null;
} else if (cached) {
@SuppressWarnings({ "unchecked", "rawtypes" })
SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(
getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
this.resettableInputs[i] = iter;
this.inputs[i] = iter;
}
}
}
Example 19
| Source File: ReduceCombineDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void prepare() throws Exception {
final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
strategy = taskContext.getTaskConfig().getDriverStrategy();
// instantiate the serializer / comparator
final TypeSerializerFactory<T> serializerFactory = taskContext.getInputSerializer(0);
comparator = taskContext.getDriverComparator(0);
serializer = serializerFactory.getSerializer();
reducer = taskContext.getStub();
output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
MemoryManager memManager = taskContext.getMemoryManager();
final int numMemoryPages = memManager.computeNumberOfPages(
taskContext.getTaskConfig().getRelativeMemoryDriver());
memory = memManager.allocatePages(taskContext.getContainingTask(), numMemoryPages);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("ReduceCombineDriver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
switch (strategy) {
case SORTED_PARTIAL_REDUCE:
// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
if (comparator.supportsSerializationWithKeyNormalization() &&
serializer.getLength() > 0 && serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) {
sorter = new FixedLengthRecordSorter<T>(serializer, comparator.duplicate(), memory);
} else {
sorter = new NormalizedKeySorter<T>(serializer, comparator.duplicate(), memory);
}
break;
case HASHED_PARTIAL_REDUCE:
table = new InPlaceMutableHashTable<T>(serializer, comparator, memory);
reduceFacade = table.new ReduceFacade(reducer, output, objectReuseEnabled);
break;
default:
throw new Exception("Invalid strategy " + taskContext.getTaskConfig().getDriverStrategy() + " for reduce combiner.");
}
}
Example 20
| Source File: LeftOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
