org.apache.flink.api.common.typeutils.TypePairComparatorFactory Java Examples
The following examples show how to use
org.apache.flink.api.common.typeutils.TypePairComparatorFactory.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TaskConfig.java From flink with Apache License 2.0 | 6 votes |
|
public <T1, T2> TypePairComparatorFactory<T1, T2> getPairComparatorFactory(ClassLoader cl) {
final String className = this.config.getString(DRIVER_PAIR_COMPARATOR_FACTORY, null);
if (className == null) {
return null;
}
@SuppressWarnings("unchecked")
final Class<TypePairComparatorFactory<T1, T2>> superClass = (Class<TypePairComparatorFactory<T1, T2>>) (Class<?>) TypePairComparatorFactory.class;
try {
final Class<? extends TypePairComparatorFactory<T1, T2>> clazz = Class.forName(className, true, cl).asSubclass(superClass);
return InstantiationUtil.instantiate(clazz, superClass);
}
catch (ClassNotFoundException cnfex) {
throw new RuntimeException("The class '" + className + "', noted in the configuration as " +
"pair comparator factory, could not be found. It is not part of the user code's class loader resources.");
}
catch (ClassCastException ccex) {
throw new CorruptConfigurationException("The class noted in the configuration as the pair comparator factory " +
"is no subclass of TypePairComparatorFactory.");
}
}
Example #2
| Source File: TaskConfig.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
|
public <T1, T2> TypePairComparatorFactory<T1, T2> getPairComparatorFactory(ClassLoader cl) {
final String className = this.config.getString(DRIVER_PAIR_COMPARATOR_FACTORY, null);
if (className == null) {
return null;
}
@SuppressWarnings("unchecked")
final Class<TypePairComparatorFactory<T1, T2>> superClass = (Class<TypePairComparatorFactory<T1, T2>>) (Class<?>) TypePairComparatorFactory.class;
try {
final Class<? extends TypePairComparatorFactory<T1, T2>> clazz = Class.forName(className, true, cl).asSubclass(superClass);
return InstantiationUtil.instantiate(clazz, superClass);
}
catch (ClassNotFoundException cnfex) {
throw new RuntimeException("The class '" + className + "', noted in the configuration as " +
"pair comparator factory, could not be found. It is not part of the user code's class loader resources.");
}
catch (ClassCastException ccex) {
throw new CorruptConfigurationException("The class noted in the configuration as the pair comparator factory " +
"is no subclass of TypePairComparatorFactory.");
}
}
Example #3
| Source File: TaskConfig.java From flink with Apache License 2.0 | 6 votes |
|
public <T1, T2> TypePairComparatorFactory<T1, T2> getPairComparatorFactory(ClassLoader cl) {
final String className = this.config.getString(DRIVER_PAIR_COMPARATOR_FACTORY, null);
if (className == null) {
return null;
}
@SuppressWarnings("unchecked")
final Class<TypePairComparatorFactory<T1, T2>> superClass = (Class<TypePairComparatorFactory<T1, T2>>) (Class<?>) TypePairComparatorFactory.class;
try {
final Class<? extends TypePairComparatorFactory<T1, T2>> clazz = Class.forName(className, true, cl).asSubclass(superClass);
return InstantiationUtil.instantiate(clazz, superClass);
}
catch (ClassNotFoundException cnfex) {
throw new RuntimeException("The class '" + className + "', noted in the configuration as " +
"pair comparator factory, could not be found. It is not part of the user code's class loader resources.");
}
catch (ClassCastException ccex) {
throw new CorruptConfigurationException("The class noted in the configuration as the pair comparator factory " +
"is no subclass of TypePairComparatorFactory.");
}
}
Example #4
| Source File: JavaApiPostPass.java From flink with Apache License 2.0 | 5 votes |
|
private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T1> info1 = (TupleTypeInfo<T1>) typeInfo1;
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T2> info2 = (TupleTypeInfo<T2>) typeInfo2;
return new RuntimePairComparatorFactory<T1,T2>();
}
Example #5
| Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #6
| Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #7
| Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #8
| Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #9
| Source File: JavaApiPostPass.java From flink with Apache License 2.0 | 5 votes |
|
private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T1> info1 = (TupleTypeInfo<T1>) typeInfo1;
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T2> info2 = (TupleTypeInfo<T2>) typeInfo2;
return new RuntimePairComparatorFactory<T1,T2>();
}
Example #10
| Source File: JavaApiPostPass.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
private static <T1 extends Tuple, T2 extends Tuple> TypePairComparatorFactory<T1,T2> createPairComparator(TypeInformation<?> typeInfo1, TypeInformation<?> typeInfo2) {
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T1> info1 = (TupleTypeInfo<T1>) typeInfo1;
// @SuppressWarnings("unchecked")
// TupleTypeInfo<T2> info2 = (TupleTypeInfo<T2>) typeInfo2;
return new RuntimePairComparatorFactory<T1,T2>();
}
Example #11
| Source File: AbstractOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #12
| Source File: AbstractOuterJoinDriver.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
|
protected abstract JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator( DriverStrategy driverStrategy, MutableObjectIterator<IT1> in1, MutableObjectIterator<IT2> in2, TypeSerializer<IT1> serializer1, TypeComparator<IT1> comparator1, TypeSerializer<IT2> serializer2, TypeComparator<IT2> comparator2, TypePairComparatorFactory<IT1, IT2> pairComparatorFactory, MemoryManager memoryManager, IOManager ioManager, double driverMemFraction ) throws Exception;
Example #13
| Source File: LeftOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
Example #14
| Source File: JoinWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
@SuppressWarnings("unchecked")
public void initialize() {
final TypeSerializer<IT1> solutionSetSerializer;
final TypeComparator<IT1> solutionSetComparator;
// grab a handle to the hash table from the iteration broker
if (taskContext instanceof AbstractIterativeTask) {
AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
String identifier = iterativeTaskContext.brokerKey();
Object table = SolutionSetBroker.instance().get(identifier);
if (table instanceof CompactingHashTable) {
this.hashTable = (CompactingHashTable<IT1>) table;
solutionSetSerializer = this.hashTable.getBuildSideSerializer();
solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
}
else if (table instanceof JoinHashMap) {
this.objectMap = (JoinHashMap<IT1>) table;
solutionSetSerializer = this.objectMap.getBuildSerializer();
solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
}
else {
throw new RuntimeException("Unrecognized solution set index: " + table);
}
} else {
throw new RuntimeException("The task context of this driver is no iterative task context.");
}
TaskConfig config = taskContext.getTaskConfig();
ClassLoader classLoader = taskContext.getUserCodeClassLoader();
TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
this.probeSideComparator = probeSideComparatorFactory.createComparator();
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
solutionSideRecord = solutionSetSerializer.createInstance();
probeSideRecord = probeSideSerializer.createInstance();
}
TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
Example #15
| Source File: LeftOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case LEFT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.LEFT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case LEFT_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
case LEFT_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
default:
throw new Exception("Unsupported driver strategy for left outer join driver: " + driverStrategy.name());
}
}
Example #16
| Source File: JoinWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
final TypeSerializer<IT2> solutionSetSerializer;
final TypeComparator<IT2> solutionSetComparator;
// grab a handle to the hash table from the iteration broker
if (taskContext instanceof AbstractIterativeTask) {
AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
String identifier = iterativeTaskContext.brokerKey();
Object table = SolutionSetBroker.instance().get(identifier);
if (table instanceof CompactingHashTable) {
this.hashTable = (CompactingHashTable<IT2>) table;
solutionSetSerializer = this.hashTable.getBuildSideSerializer();
solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
}
else if (table instanceof JoinHashMap) {
this.objectMap = (JoinHashMap<IT2>) table;
solutionSetSerializer = this.objectMap.getBuildSerializer();
solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
}
else {
throw new RuntimeException("Unrecognized solution set index: " + table);
}
}
else {
throw new Exception("The task context of this driver is no iterative task context.");
}
TaskConfig config = taskContext.getTaskConfig();
ClassLoader classLoader = taskContext.getUserCodeClassLoader();
TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
this.probeSideComparator = probeSideComparatorFactory.createComparator();
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
solutionSideRecord = solutionSetSerializer.createInstance();
probeSideRecord = probeSideSerializer.createInstance();
}
TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
Example #17
| Source File: CoGroupWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
@SuppressWarnings("unchecked")
public void initialize() throws Exception {
final TypeComparator<IT2> solutionSetComparator;
// grab a handle to the hash table from the iteration broker
if (taskContext instanceof AbstractIterativeTask) {
AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
String identifier = iterativeTaskContext.brokerKey();
Object table = SolutionSetBroker.instance().get(identifier);
if (table instanceof CompactingHashTable) {
this.hashTable = (CompactingHashTable<IT2>) table;
solutionSetSerializer = this.hashTable.getBuildSideSerializer();
solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
}
else if (table instanceof JoinHashMap) {
this.objectMap = (JoinHashMap<IT2>) table;
solutionSetSerializer = this.objectMap.getBuildSerializer();
solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
}
else {
throw new RuntimeException("Unrecognized solution set index: " + table);
}
}
else {
throw new Exception("The task context of this driver is no iterative task context.");
}
TaskConfig config = taskContext.getTaskConfig();
ClassLoader classLoader = taskContext.getUserCodeClassLoader();
TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
this.probeSideComparator = probeSideComparatorFactory.createComparator();
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
solutionSideRecord = solutionSetSerializer.createInstance();
};
TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator);
}
Example #18
| Source File: DualInputPlanNode.java From flink with Apache License 2.0 | 4 votes |
|
public void setPairComparator(TypePairComparatorFactory<?, ?> comparator) {
this.pairComparator = comparator;
}
Example #19
| Source File: DualInputPlanNode.java From flink with Apache License 2.0 | 4 votes |
|
public TypePairComparatorFactory<?, ?> getPairComparator() {
return this.pairComparator;
}
Example #20
| Source File: AbstractOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void prepare() throws Exception {
final TaskConfig config = this.taskContext.getTaskConfig();
// obtain task manager's memory manager and I/O manager
final MemoryManager memoryManager = this.taskContext.getMemoryManager();
final IOManager ioManager = this.taskContext.getIOManager();
// set up memory and I/O parameters
final double driverMemFraction = config.getRelativeMemoryDriver();
final DriverStrategy ls = config.getDriverStrategy();
final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
// get serializers and comparators
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for outer join driver");
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
// create and return outer join iterator according to provided local strategy.
if (objectReuseEnabled) {
this.outerJoinIterator = getReusingOuterJoinIterator(
ls,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory,
memoryManager,
ioManager,
driverMemFraction
);
} else {
this.outerJoinIterator = getNonReusingOuterJoinIterator(
ls,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory,
memoryManager,
ioManager,
driverMemFraction
);
}
this.outerJoinIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
}
}
Example #21
| Source File: AbstractCachedBuildSideJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void initialize() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
MutableObjectIterator<IT1> input1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
MutableObjectIterator<IT2> input2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory =
this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
double availableMemory = config.getRelativeMemoryDriver();
boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration()
.getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
if (buildSideIndex == 0 && probeSideIndex == 1) {
matchIterator = new ReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else if (buildSideIndex == 1 && probeSideIndex == 0) {
matchIterator = new ReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else {
throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
}
} else {
if (buildSideIndex == 0 && probeSideIndex == 1) {
matchIterator = new NonReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else if (buildSideIndex == 1 && probeSideIndex == 0) {
matchIterator = new NonReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else {
throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
}
}
this.matchIterator.open();
}
Example #22
| Source File: CoGroupWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
@SuppressWarnings("unchecked")
public void initialize() {
final TypeComparator<IT1> solutionSetComparator;
// grab a handle to the hash table from the iteration broker
if (taskContext instanceof AbstractIterativeTask) {
AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
String identifier = iterativeTaskContext.brokerKey();
Object table = SolutionSetBroker.instance().get(identifier);
if (table instanceof CompactingHashTable) {
this.hashTable = (CompactingHashTable<IT1>) table;
solutionSetSerializer = this.hashTable.getBuildSideSerializer();
solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
}
else if (table instanceof JoinHashMap) {
this.objectMap = (JoinHashMap<IT1>) table;
solutionSetSerializer = this.objectMap.getBuildSerializer();
solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
}
else {
throw new RuntimeException("Unrecognized solution set index: " + table);
}
} else {
throw new RuntimeException("The task context of this driver is no iterative task context.");
}
TaskConfig config = taskContext.getTaskConfig();
ClassLoader classLoader = taskContext.getUserCodeClassLoader();
TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
this.probeSideComparator = probeSideComparatorFactory.createComparator();
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
solutionSideRecord = solutionSetSerializer.createInstance();
}
TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
Example #23
| Source File: CoGroupDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void prepare() throws Exception
{
final TaskConfig config = this.taskContext.getTaskConfig();
if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
}
final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
// get the key positions and types
final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(
this.taskContext.getUserCodeClassLoader());
if (pairComparatorFactory == null) {
throw new Exception("Missing pair comparator factory for CoGroup driver");
}
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (LOG.isDebugEnabled()) {
LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
}
if (objectReuseEnabled) {
// create CoGroupTaskIterator according to provided local strategy.
this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(
in1, in2,
serializer1, groupComparator1,
serializer2, groupComparator2,
pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
} else {
// create CoGroupTaskIterator according to provided local strategy.
this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(
in1, in2,
serializer1, groupComparator1,
serializer2, groupComparator2,
pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
}
// open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
this.coGroupIterator.open();
if (LOG.isDebugEnabled()) {
LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
}
}
Example #24
| Source File: FullOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case FULL_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.FULL,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case FULL_OUTER_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
case FULL_OUTER_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
default:
throw new Exception("Unsupported driver strategy for full outer join driver: " + driverStrategy.name());
}
}
Example #25
| Source File: FullOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case FULL_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.FULL,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case FULL_OUTER_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
case FULL_OUTER_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
true,
false);
default:
throw new Exception("Unsupported driver strategy for full outer join driver: " + driverStrategy.name());
}
}
Example #26
| Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
|
public void setDriverPairComparator(TypePairComparatorFactory<?, ?> factory) {
final Class<?> clazz = factory.getClass();
InstantiationUtil.checkForInstantiation(clazz);
this.config.setString(DRIVER_PAIR_COMPARATOR_FACTORY, clazz.getName());
}
Example #27
| Source File: AbstractCachedBuildSideJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public void initialize() throws Exception {
TaskConfig config = this.taskContext.getTaskConfig();
final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
MutableObjectIterator<IT1> input1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
MutableObjectIterator<IT2> input2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory =
this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
double availableMemory = config.getRelativeMemoryDriver();
boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration()
.getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
if (buildSideIndex == 0 && probeSideIndex == 1) {
matchIterator = new ReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else if (buildSideIndex == 1 && probeSideIndex == 0) {
matchIterator = new ReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else {
throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
}
} else {
if (buildSideIndex == 0 && probeSideIndex == 1) {
matchIterator = new NonReusingBuildFirstReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else if (buildSideIndex == 1 && probeSideIndex == 0) {
matchIterator = new NonReusingBuildSecondReOpenableHashJoinIterator<IT1, IT2, OT>(
input1, input2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
this.taskContext.getMemoryManager(),
this.taskContext.getIOManager(),
this.taskContext.getContainingTask(),
availableMemory,
false,
false,
hashJoinUseBitMaps);
} else {
throw new Exception("Error: Inconsistent setup for repeatable hash join driver.");
}
}
this.matchIterator.open();
}
Example #28
| Source File: RightOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case RIGHT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new ReusingMergeOuterJoinIterator<>(
OuterJoinType.RIGHT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case RIGHT_HYBRIDHASH_BUILD_FIRST:
return new ReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
case RIGHT_HYBRIDHASH_BUILD_SECOND:
return new ReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
default:
throw new Exception("Unsupported driver strategy for right outer join driver: " + driverStrategy.name());
}
}
Example #29
| Source File: RightOuterJoinDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
protected JoinTaskIterator<IT1, IT2, OT> getNonReusingOuterJoinIterator(
DriverStrategy driverStrategy,
MutableObjectIterator<IT1> in1,
MutableObjectIterator<IT2> in2,
TypeSerializer<IT1> serializer1,
TypeComparator<IT1> comparator1,
TypeSerializer<IT2> serializer2,
TypeComparator<IT2> comparator2,
TypePairComparatorFactory<IT1, IT2> pairComparatorFactory,
MemoryManager memoryManager,
IOManager ioManager,
double driverMemFraction
) throws Exception {
switch (driverStrategy) {
case RIGHT_OUTER_MERGE:
int numPages = memoryManager.computeNumberOfPages(driverMemFraction);
return new NonReusingMergeOuterJoinIterator<>(
OuterJoinType.RIGHT,
in1,
in2,
serializer1,
comparator1,
serializer2,
comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager,
ioManager,
numPages,
super.taskContext.getContainingTask()
);
case RIGHT_HYBRIDHASH_BUILD_FIRST:
return new NonReusingBuildFirstHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator21(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
true,
false,
false);
case RIGHT_HYBRIDHASH_BUILD_SECOND:
return new NonReusingBuildSecondHashJoinIterator<>(in1, in2,
serializer1, comparator1,
serializer2, comparator2,
pairComparatorFactory.createComparator12(comparator1, comparator2),
memoryManager, ioManager,
this.taskContext.getContainingTask(),
driverMemFraction,
false,
true,
false);
default:
throw new Exception("Unsupported driver strategy for right outer join driver: " + driverStrategy.name());
}
}
Example #30
| Source File: JoinWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
|
@Override
@SuppressWarnings("unchecked")
public void initialize() {
final TypeSerializer<IT1> solutionSetSerializer;
final TypeComparator<IT1> solutionSetComparator;
// grab a handle to the hash table from the iteration broker
if (taskContext instanceof AbstractIterativeTask) {
AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext;
String identifier = iterativeTaskContext.brokerKey();
Object table = SolutionSetBroker.instance().get(identifier);
if (table instanceof CompactingHashTable) {
this.hashTable = (CompactingHashTable<IT1>) table;
solutionSetSerializer = this.hashTable.getBuildSideSerializer();
solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
}
else if (table instanceof JoinHashMap) {
this.objectMap = (JoinHashMap<IT1>) table;
solutionSetSerializer = this.objectMap.getBuildSerializer();
solutionSetComparator = this.objectMap.getBuildComparator().duplicate();
}
else {
throw new RuntimeException("Unrecognized solution set index: " + table);
}
} else {
throw new RuntimeException("The task context of this driver is no iterative task context.");
}
TaskConfig config = taskContext.getTaskConfig();
ClassLoader classLoader = taskContext.getUserCodeClassLoader();
TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer();
TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
this.probeSideComparator = probeSideComparatorFactory.createComparator();
ExecutionConfig executionConfig = taskContext.getExecutionConfig();
objectReuseEnabled = executionConfig.isObjectReuseEnabled();
if (objectReuseEnabled) {
solutionSideRecord = solutionSetSerializer.createInstance();
probeSideRecord = probeSideSerializer.createInstance();
}
TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader());
pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator);
}
