org.apache.flink.api.common.typeutils.TypeComparatorFactory Java Examples
The following examples show how to use
org.apache.flink.api.common.typeutils.TypeComparatorFactory.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java From flink with Apache License 2.0 | 6 votes |
/** * Creates all the serializers and comparators. */ protected void initInputsSerializersAndComparators(int numInputs, int numComparators) throws Exception { this.inputSerializers = new TypeSerializerFactory<?>[numInputs]; this.inputComparators = numComparators > 0 ? new TypeComparator<?>[numComparators] : null; this.inputIterators = new MutableObjectIterator<?>[numInputs]; ClassLoader userCodeClassLoader = getUserCodeClassLoader(); for (int i = 0; i < numInputs; i++) { final TypeSerializerFactory<?> serializerFactory = this.config.getInputSerializer(i, userCodeClassLoader); this.inputSerializers[i] = serializerFactory; this.inputIterators[i] = createInputIterator(this.inputReaders[i], this.inputSerializers[i]); } // ---------------- create the driver's comparators --------------------- for (int i = 0; i < numComparators; i++) { if (this.inputComparators != null) { final TypeComparatorFactory<?> comparatorFactory = this.config.getDriverComparator(i, userCodeClassLoader); this.inputComparators[i] = comparatorFactory.createComparator(); } } }
Example #2
Source File: Utils.java From flink with Apache License 2.0 | 6 votes |
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<>(comparator); }
Example #3
Source File: Utils.java From flink with Apache License 2.0 | 6 votes |
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<>(comparator); }
Example #4
Source File: JavaApiPostPass.java From flink with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<T>(comparator); }
Example #5
Source File: Utils.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) { TypeComparator<T> comparator; if (typeInfo instanceof CompositeType) { comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig); } else if (typeInfo instanceof AtomicType) { // handle grouping of atomic types comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig); } else { throw new RuntimeException("Unrecognized type: " + typeInfo); } return new RuntimeComparatorFactory<>(comparator); }
Example #6
Source File: IterationHeadTask.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private <BT> JoinHashMap<BT> initJoinHashMap() { TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer (getUserCodeClassLoader()); TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator (getUserCodeClassLoader()); TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer(); TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator(); return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator); }
Example #7
Source File: BatchTask.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private <T> TypeComparator<T> getLocalStrategyComparator(int inputNum) throws Exception { TypeComparatorFactory<T> compFact = this.config.getInputComparator(inputNum, getUserCodeClassLoader()); if (compFact == null) { throw new Exception("Missing comparator factory for local strategy on input " + inputNum); } return compFact.createComparator(); }
Example #8
Source File: SingleInputPlanNode.java From flink with Apache License 2.0 | 5 votes |
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders) { super(template, nodeName, driverStrategy); this.input = input; this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()]; this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()]; this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][]; if(driverStrategy.getNumRequiredComparators() > 0) { this.driverKeys[0] = driverKeyFields; this.driverSortOrders[0] = driverSortOrders; } if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input.setReplicationFactor(getParallelism()); } final PlanNode predNode = input.getSource(); if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) { if (this.branchPlan == null) { this.branchPlan = new HashMap<OptimizerNode, PlanNode>(); } this.branchPlan.putAll(predNode.branchPlan); } }
Example #9
Source File: IterationHeadTask.java From flink with Apache License 2.0 | 5 votes |
private <BT> JoinHashMap<BT> initJoinHashMap() { TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer (getUserCodeClassLoader()); TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator (getUserCodeClassLoader()); TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer(); TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator(); return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator); }
Example #10
Source File: TaskConfig.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory, String classNameKey, String parametersPrefix) { // sanity check the factory type InstantiationUtil.checkForInstantiation(factory.getClass()); // store the type this.config.setString(classNameKey, factory.getClass().getName()); // store the parameters final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix); factory.writeParametersToConfig(parameters); }
Example #11
Source File: Utils.java From flink with Apache License 2.0 | 5 votes |
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) { PlanNode source = channel.getSource(); Operator<?> javaOp = source.getProgramOperator(); TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType(); return createComparator(type, channel.getShipStrategyKeys(), getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig); }
Example #12
Source File: TaskConfig.java From flink with Apache License 2.0 | 5 votes |
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory, String classNameKey, String parametersPrefix) { // sanity check the factory type InstantiationUtil.checkForInstantiation(factory.getClass()); // store the type this.config.setString(classNameKey, factory.getClass().getName()); // store the parameters final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix); factory.writeParametersToConfig(parameters); }
Example #13
Source File: SynchronousChainedCombineDriver.java From flink with Apache License 2.0 | 5 votes |
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.combiner, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
Example #14
Source File: SingleInputPlanNode.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders) { super(template, nodeName, driverStrategy); this.input = input; this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()]; this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()]; this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][]; if(driverStrategy.getNumRequiredComparators() > 0) { this.driverKeys[0] = driverKeyFields; this.driverSortOrders[0] = driverSortOrders; } if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) { this.input.setReplicationFactor(getParallelism()); } final PlanNode predNode = input.getSource(); if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) { if (this.branchPlan == null) { this.branchPlan = new HashMap<OptimizerNode, PlanNode>(); } this.branchPlan.putAll(predNode.branchPlan); } }
Example #15
Source File: GroupCombineChainedDriver.java From flink with Apache License 2.0 | 5 votes |
@Override public void openTask() throws Exception { // open the stub first final Configuration stubConfig = this.config.getStubParameters(); BatchTask.openUserCode(this.reducer, stubConfig); // ----------------- Set up the sorter ------------------------- // instantiate the serializer / comparator final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader); final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader); this.serializer = serializerFactory.getSerializer(); TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator(); this.groupingComparator = groupingComparatorFactory.createComparator(); MemoryManager memManager = this.parent.getEnvironment().getMemoryManager(); final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver()); this.memory = memManager.allocatePages(this.parent, numMemoryPages); // instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter if (sortingComparator.supportsSerializationWithKeyNormalization() && this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING) { this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } else { this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory); } if (LOG.isDebugEnabled()) { LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + "."); } }
Example #16
Source File: Utils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) { PlanNode source = channel.getSource(); Operator<?> javaOp = source.getProgramOperator(); TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType(); return createComparator(type, channel.getShipStrategyKeys(), getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig); }
Example #17
Source File: TaskConfig.java From flink with Apache License 2.0 | 5 votes |
private void setTypeComparatorFactory(TypeComparatorFactory<?> factory, String classNameKey, String parametersPrefix) { // sanity check the factory type InstantiationUtil.checkForInstantiation(factory.getClass()); // store the type this.config.setString(classNameKey, factory.getClass().getName()); // store the parameters final DelegatingConfiguration parameters = new DelegatingConfiguration(this.config, parametersPrefix); factory.writeParametersToConfig(parameters); }
Example #18
Source File: IterationHeadTask.java From flink with Apache License 2.0 | 5 votes |
private <BT> JoinHashMap<BT> initJoinHashMap() { TypeSerializerFactory<BT> solutionTypeSerializerFactory = config.getSolutionSetSerializer (getUserCodeClassLoader()); TypeComparatorFactory<BT> solutionTypeComparatorFactory = config.getSolutionSetComparator (getUserCodeClassLoader()); TypeSerializer<BT> solutionTypeSerializer = solutionTypeSerializerFactory.getSerializer(); TypeComparator<BT> solutionTypeComparator = solutionTypeComparatorFactory.createComparator(); return new JoinHashMap<BT>(solutionTypeSerializer, solutionTypeComparator); }
Example #19
Source File: BatchTask.java From flink with Apache License 2.0 | 5 votes |
private <T> TypeComparator<T> getLocalStrategyComparator(int inputNum) throws Exception { TypeComparatorFactory<T> compFact = this.config.getInputComparator(inputNum, getUserCodeClassLoader()); if (compFact == null) { throw new Exception("Missing comparator factory for local strategy on input " + inputNum); } return compFact.createComparator(); }
Example #20
Source File: JoinWithSolutionSetSecondDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() throws Exception { final TypeSerializer<IT2> solutionSetSerializer; final TypeComparator<IT2> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT2>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT2>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new Exception("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer(); TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); probeSideRecord = probeSideSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator12(this.probeSideComparator, solutionSetComparator); }
Example #21
Source File: CoGroupWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #22
Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
public void setSolutionSetComparator(TypeComparatorFactory<?> factory) { setTypeComparatorFactory(factory, ITERATION_SOLUTION_SET_COMPARATOR, ITERATION_SOLUTION_SET_COMPARATOR_PARAMETERS); }
Example #23
Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
public <T> TypeComparatorFactory<T> getSolutionSetComparator(ClassLoader cl) { return getTypeComparatorFactory(ITERATION_SOLUTION_SET_COMPARATOR, ITERATION_SOLUTION_SET_COMPARATOR_PARAMETERS, cl); }
Example #24
Source File: CoGroupWithSolutionSetFirstDriver.java From flink with Apache License 2.0 | 4 votes |
@Override @SuppressWarnings("unchecked") public void initialize() { final TypeComparator<IT1> solutionSetComparator; // grab a handle to the hash table from the iteration broker if (taskContext instanceof AbstractIterativeTask) { AbstractIterativeTask<?, ?> iterativeTaskContext = (AbstractIterativeTask<?, ?>) taskContext; String identifier = iterativeTaskContext.brokerKey(); Object table = SolutionSetBroker.instance().get(identifier); if (table instanceof CompactingHashTable) { this.hashTable = (CompactingHashTable<IT1>) table; solutionSetSerializer = this.hashTable.getBuildSideSerializer(); solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate(); } else if (table instanceof JoinHashMap) { this.objectMap = (JoinHashMap<IT1>) table; solutionSetSerializer = this.objectMap.getBuildSerializer(); solutionSetComparator = this.objectMap.getBuildComparator().duplicate(); } else { throw new RuntimeException("Unrecognized solution set index: " + table); } } else { throw new RuntimeException("The task context of this driver is no iterative task context."); } TaskConfig config = taskContext.getTaskConfig(); ClassLoader classLoader = taskContext.getUserCodeClassLoader(); TypeComparatorFactory<IT2> probeSideComparatorFactory = config.getDriverComparator(0, classLoader); this.probeSideSerializer = taskContext.<IT2>getInputSerializer(0).getSerializer(); this.probeSideComparator = probeSideComparatorFactory.createComparator(); ExecutionConfig executionConfig = taskContext.getExecutionConfig(); objectReuseEnabled = executionConfig.isObjectReuseEnabled(); if (objectReuseEnabled) { solutionSideRecord = solutionSetSerializer.createInstance(); } TypePairComparatorFactory<IT1, IT2> factory = taskContext.getTaskConfig().getPairComparatorFactory(taskContext.getUserCodeClassLoader()); pairComparator = factory.createComparator21(solutionSetComparator, this.probeSideComparator); }
Example #25
Source File: BatchTask.java From flink with Apache License 2.0 | 4 votes |
/** * Creates the {@link Collector} for the given task, as described by the given configuration. The * output collector contains the writers that forward the data to the different tasks that the given task * is connected to. Each writer applies the partitioning as described in the configuration. * * @param task The task that the output collector is created for. * @param config The configuration describing the output shipping strategies. * @param cl The classloader used to load user defined types. * @param eventualOutputs The output writers that this task forwards to the next task for each output. * @param outputOffset The offset to start to get the writers for the outputs * @param numOutputs The number of outputs described in the configuration. * * @return The OutputCollector that data produced in this task is submitted to. */ public static <T> Collector<T> getOutputCollector(AbstractInvokable task, TaskConfig config, ClassLoader cl, List<RecordWriter<?>> eventualOutputs, int outputOffset, int numOutputs) throws Exception { if (numOutputs == 0) { return null; } // get the factory for the serializer final TypeSerializerFactory<T> serializerFactory = config.getOutputSerializer(cl); final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<>(numOutputs); // create a writer for each output for (int i = 0; i < numOutputs; i++) { // create the OutputEmitter from output ship strategy final ShipStrategyType strategy = config.getOutputShipStrategy(i); final int indexInSubtaskGroup = task.getIndexInSubtaskGroup(); final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl); final ChannelSelector<SerializationDelegate<T>> oe; if (compFactory == null) { oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup); } else { final DataDistribution dataDist = config.getOutputDataDistribution(i, cl); final Partitioner<?> partitioner = config.getOutputPartitioner(i, cl); final TypeComparator<T> comparator = compFactory.createComparator(); oe = new OutputEmitter<T>(strategy, indexInSubtaskGroup, comparator, partitioner, dataDist); } final RecordWriter<SerializationDelegate<T>> recordWriter = new RecordWriterBuilder() .setChannelSelector(oe) .setTaskName(task.getEnvironment().getTaskInfo().getTaskName()) .build(task.getEnvironment().getWriter(outputOffset + i)); recordWriter.setMetricGroup(task.getEnvironment().getMetricGroup().getIOMetricGroup()); writers.add(recordWriter); } if (eventualOutputs != null) { eventualOutputs.addAll(writers); } return new OutputCollector<T>(writers, serializerFactory.getSerializer()); }
Example #26
Source File: AssignRangeIndex.java From flink with Apache License 2.0 | 4 votes |
public AssignRangeIndex(TypeComparatorFactory<IN> typeComparator) { this.typeComparator = typeComparator; }
Example #27
Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
public <T> TypeComparatorFactory<T> getInputComparator(int inputNum, ClassLoader cl) { return getTypeComparatorFactory(INPUT_STRATEGY_COMPARATOR_FACTORY_PREFIX + inputNum, INPUT_STRATEGY_COMPARATOR_PARAMETERS_PREFIX + inputNum + SEPARATOR, cl); }
Example #28
Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
public void setOutputComparator(TypeComparatorFactory<?> factory, int outputNum) { setTypeComparatorFactory(factory, OUTPUT_TYPE_COMPARATOR_FACTORY_PREFIX + outputNum, OUTPUT_TYPE_COMPARATOR_PARAMETERS_PREFIX + outputNum + SEPARATOR); }
Example #29
Source File: DualInputPlanNode.java From flink with Apache License 2.0 | 4 votes |
public TypeComparatorFactory<?> getComparator2() { return this.comparator2; }
Example #30
Source File: TaskConfig.java From flink with Apache License 2.0 | 4 votes |
public void setDriverComparator(TypeComparatorFactory<?> factory, int inputNum) { setTypeComparatorFactory(factory, DRIVER_COMPARATOR_FACTORY_PREFIX + inputNum, DRIVER_COMPARATOR_PARAMETERS_PREFIX + inputNum + SEPARATOR); }