org.apache.flink.runtime.iterative.concurrent.SolutionSetBroker Java Examples
The following examples show how to use
org.apache.flink.runtime.iterative.concurrent.SolutionSetBroker.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractIterativeTask.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Creates a new solution set update output collector. * * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing) * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing) * {@link SolutionSetUpdateOutputCollector} is created. * * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call * collect(T) of the delegate. * * @param delegate null -OR- a delegate collector to be called by the newly created collector * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or * {@link SolutionSetUpdateOutputCollector} */ protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) { Broker<Object> solutionSetBroker = SolutionSetBroker.instance(); Object ss = solutionSetBroker.get(brokerKey()); if (ss instanceof CompactingHashTable) { @SuppressWarnings("unchecked") CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss; return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate); } else if (ss instanceof JoinHashMap) { @SuppressWarnings("unchecked") JoinHashMap<OT> map = (JoinHashMap<OT>) ss; return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate); } else { throw new RuntimeException("Unrecognized solution set handle: " + ss); } }
Example #2
Source File: AbstractIterativeTask.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a new solution set update output collector. * * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing) * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing) * {@link SolutionSetUpdateOutputCollector} is created. * * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call * collect(T) of the delegate. * * @param delegate null -OR- a delegate collector to be called by the newly created collector * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or * {@link SolutionSetUpdateOutputCollector} */ protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) { Broker<Object> solutionSetBroker = SolutionSetBroker.instance(); Object ss = solutionSetBroker.get(brokerKey()); if (ss instanceof CompactingHashTable) { @SuppressWarnings("unchecked") CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss; return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate); } else if (ss instanceof JoinHashMap) { @SuppressWarnings("unchecked") JoinHashMap<OT> map = (JoinHashMap<OT>) ss; return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate); } else { throw new RuntimeException("Unrecognized solution set handle: " + ss); } }
Example #3
Source File: AbstractIterativeTask.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a new solution set update output collector. * * <p>This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the * solution set of workset iterations. Depending on the task configuration, either a fast (non-probing) * {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or normal (re-probing) * {@link SolutionSetUpdateOutputCollector} is created. * * <p>If a non-null delegate is given, the new {@link Collector} will write back to the solution set and also call * collect(T) of the delegate. * * @param delegate null -OR- a delegate collector to be called by the newly created collector * @return a new {@link org.apache.flink.runtime.iterative.io.SolutionSetFastUpdateOutputCollector} or * {@link SolutionSetUpdateOutputCollector} */ protected Collector<OT> createSolutionSetUpdateOutputCollector(Collector<OT> delegate) { Broker<Object> solutionSetBroker = SolutionSetBroker.instance(); Object ss = solutionSetBroker.get(brokerKey()); if (ss instanceof CompactingHashTable) { @SuppressWarnings("unchecked") CompactingHashTable<OT> solutionSet = (CompactingHashTable<OT>) ss; return new SolutionSetUpdateOutputCollector<OT>(solutionSet, delegate); } else if (ss instanceof JoinHashMap) { @SuppressWarnings("unchecked") JoinHashMap<OT> map = (JoinHashMap<OT>) ss; return new SolutionSetObjectsUpdateOutputCollector<OT>(map, delegate); } else { throw new RuntimeException("Unrecognized solution set handle: " + ss); } }
Example #4
Source File: CoGroupWithSolutionSetSecondDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); }; TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #5
Source File: JoinWithSolutionSetSecondDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeSerializer<IT2> solutionSetSerializer; final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #6
Source File: CoGroupWithSolutionSetFirstDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #7
Source File: JoinWithSolutionSetFirstDriver.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeSerializer<IT1> solutionSetSerializer; final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #8
Source File: CoGroupWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); }; TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #9
Source File: JoinWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeSerializer<IT2> solutionSetSerializer; final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #10
Source File: CoGroupWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #11
Source File: JoinWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeSerializer<IT1> solutionSetSerializer; final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #12
Source File: CoGroupWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); }; TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #13
Source File: JoinWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeSerializer<IT2> solutionSetSerializer; final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #14
Source File: CoGroupWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #15
Source File: JoinWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeSerializer<IT1> solutionSetSerializer; final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT2> probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }