org.apache.flink.api.java.typeutils.ListTypeInfo Java Examples
The following examples show how to use
org.apache.flink.api.java.typeutils.ListTypeInfo.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ProcTimeRangeBoundedPrecedingFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRowData(); // input element are all binary row as they are came from network RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes); // we keep the elements received in a map state indexed based on their ingestion time ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<>(inputType); MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>( "inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo); inputState = getRuntimeContext().getMapState(mapStateDescriptor); RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes); ValueStateDescriptor<RowData> stateDescriptor = new ValueStateDescriptor<RowData>("accState", accTypeInfo); accState = getRuntimeContext().getState(stateDescriptor); ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>( "cleanupTsState", Types.LONG ); this.cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor); }
Example #2
Source File: RowTimeSortOperator.java From flink with Apache License 2.0 | 6 votes |
@Override public void open() throws Exception { super.open(); LOG.info("Opening RowTimeSortOperator"); if (gComparator != null) { comparator = gComparator.newInstance(getContainingTask().getUserCodeClassLoader()); gComparator = null; } BasicTypeInfo<Long> keyTypeInfo = BasicTypeInfo.LONG_TYPE_INFO; ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>( "dataState", keyTypeInfo, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState", Long.class); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); }
Example #3
Source File: LastValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public TypeInformation<GenericRowData> getAccumulatorType() { LogicalType[] fieldTypes = new LogicalType[] { fromTypeInfoToLogicalType(getResultType()), new BigIntType(), new TypeInformationRawType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)), new TypeInformationRawType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false)) }; String[] fieldNames = new String[] { "lastValue", "lastOrder", "valueToOrderMapView", "orderToValueMapView" }; return (TypeInformation) new RowDataTypeInfo(fieldTypes, fieldNames); }
Example #4
Source File: LastValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public TypeInformation<GenericRow> getAccumulatorType() { LogicalType[] fieldTypes = new LogicalType[] { fromTypeInfoToLogicalType(getResultType()), new BigIntType(), new TypeInformationAnyType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)), new TypeInformationAnyType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false)) }; String[] fieldNames = new String[] { "lastValue", "lastOrder", "valueToOrderMapView", "orderToValueMapView" }; return (TypeInformation) new BaseRowTypeInfo(fieldTypes, fieldNames); }
Example #5
Source File: LastValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public GenericRow createAccumulator() { // The accumulator schema: // lastValue: T // lastOrder: Long // valueToOrderMap: BinaryGeneric<MapView<T, List<Long>>> // orderToValueMap: BinaryGeneric<MapView<Long, List<T>>> GenericRow acc = new GenericRow(4); acc.setField(0, null); acc.setField(1, null); acc.setField(2, new BinaryGeneric<>( new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)), getValueToOrderMapViewSerializer())); acc.setField(3, new BinaryGeneric<>( new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())), getOrderToValueMapViewSerializer())); return acc; }
Example #6
Source File: FirstValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public TypeInformation<GenericRow> getAccumulatorType() { LogicalType[] fieldTypes = new LogicalType[] { fromTypeInfoToLogicalType(getResultType()), new BigIntType(), new TypeInformationAnyType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)), new TypeInformationAnyType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false)) }; String[] fieldNames = new String[] { "firstValue", "firstOrder", "valueToOrderMapView", "orderToValueMapView" }; return (TypeInformation) new BaseRowTypeInfo(fieldTypes, fieldNames); }
Example #7
Source File: FirstValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public GenericRow createAccumulator() { // The accumulator schema: // firstValue: T // fistOrder: Long // valueToOrderMap: BinaryGeneric<MapView<T, List<Long>>> // orderToValueMap: BinaryGeneric<MapView<Long, List<T>>> GenericRow acc = new GenericRow(4); acc.setField(0, null); acc.setField(1, null); acc.setField(2, new BinaryGeneric<>( new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)), getValueToOrderMapViewSerializer())); acc.setField(3, new BinaryGeneric<>( new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())), getOrderToValueMapViewSerializer())); return acc; }
Example #8
Source File: RetractableTopNFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); // compile equaliser equaliser = generatedEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader()); generatedEqualiser = null; ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<BaseRow, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>( "data-state", sortKeyType, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); ValueStateDescriptor<SortedMap<BaseRow, Long>> valueStateDescriptor = new ValueStateDescriptor<>( "sorted-map", new SortedMapTypeInfo<>(sortKeyType, BasicTypeInfo.LONG_TYPE_INFO, serializableComparator)); treeMap = getRuntimeContext().getState(valueStateDescriptor); }
Example #9
Source File: AbstractRowTimeUnboundedPrecedingOver.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRow(); sortedTimestamps = new LinkedList<Long>(); // initialize accumulator state BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes); ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes); ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType); MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); initCleanupTimeState("RowTimeUnboundedOverCleanupTime"); }
Example #10
Source File: ProcTimeRangeBoundedPrecedingFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRow(); // input element are all binary row as they are came from network BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes); // we keep the elements received in a map state indexed based on their ingestion time ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType); MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<Long, List<BaseRow>>( "inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo); inputState = getRuntimeContext().getMapState(mapStateDescriptor); BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes); ValueStateDescriptor<BaseRow> stateDescriptor = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo); accState = getRuntimeContext().getState(stateDescriptor); initCleanupTimeState("ProcTimeBoundedRangeOverCleanupTime"); }
Example #11
Source File: AbstractRowTimeUnboundedPrecedingOver.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRowData(); sortedTimestamps = new LinkedList<Long>(); // initialize accumulator state RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes); ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes); ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType); MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); initCleanupTimeState("RowTimeUnboundedOverCleanupTime"); }
Example #12
Source File: RetractableTopNFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); // compile equaliser equaliser = generatedEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader()); generatedEqualiser = null; ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>( "data-state", sortKeyType, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); ValueStateDescriptor<SortedMap<RowData, Long>> valueStateDescriptor = new ValueStateDescriptor<>( "sorted-map", new SortedMapTypeInfo<>(sortKeyType, BasicTypeInfo.LONG_TYPE_INFO, serializableComparator)); treeMap = getRuntimeContext().getState(valueStateDescriptor); }
Example #13
Source File: RowTimeSortOperator.java From flink with Apache License 2.0 | 6 votes |
@Override public void open() throws Exception { super.open(); LOG.info("Opening RowTimeSortOperator"); if (gComparator != null) { comparator = gComparator.newInstance(getContainingTask().getUserCodeClassLoader()); gComparator = null; } BasicTypeInfo<Long> keyTypeInfo = BasicTypeInfo.LONG_TYPE_INFO; ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>( "dataState", keyTypeInfo, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<>("lastTriggeringTsState", Long.class); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); }
Example #14
Source File: FirstValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public GenericRowData createAccumulator() { // The accumulator schema: // firstValue: T // fistOrder: Long // valueToOrderMap: RawValueData<MapView<T, List<Long>>> // orderToValueMap: RawValueData<MapView<Long, List<T>>> GenericRowData acc = new GenericRowData(4); acc.setField(0, null); acc.setField(1, null); acc.setField(2, RawValueData.fromObject( new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)))); acc.setField(3, RawValueData.fromObject( new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())))); return acc; }
Example #15
Source File: FirstValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public TypeInformation<GenericRowData> getAccumulatorType() { LogicalType[] fieldTypes = new LogicalType[] { fromTypeInfoToLogicalType(getResultType()), new BigIntType(), new TypeInformationRawType<>(new MapViewTypeInfo<>(getResultType(), new ListTypeInfo<>(Types.LONG), false, false)), new TypeInformationRawType<>(new MapViewTypeInfo<>(Types.LONG, new ListTypeInfo<>(getResultType()), false, false)) }; String[] fieldNames = new String[] { "firstValue", "firstOrder", "valueToOrderMapView", "orderToValueMapView" }; return (TypeInformation) new RowDataTypeInfo(fieldTypes, fieldNames); }
Example #16
Source File: LastValueWithRetractAggFunction.java From flink with Apache License 2.0 | 6 votes |
@Override public GenericRowData createAccumulator() { // The accumulator schema: // lastValue: T // lastOrder: Long // valueToOrderMap: RawValueData<MapView<T, List<Long>>> // orderToValueMap: RawValueData<MapView<Long, List<T>>> GenericRowData acc = new GenericRowData(4); acc.setField(0, null); acc.setField(1, null); acc.setField(2, RawValueData.fromObject( new MapView<>(getResultType(), new ListTypeInfo<>(Types.LONG)))); acc.setField(3, RawValueData.fromObject( new MapView<>(Types.LONG, new ListTypeInfo<>(getResultType())))); return acc; }
Example #17
Source File: RowTimeRangeBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRowData(); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>( "lastTriggeringTsState", Types.LONG); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes); ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes); ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType); MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); ValueStateDescriptor<Long> cleanupTsStateDescriptor = new ValueStateDescriptor<>( "cleanupTsState", Types.LONG ); this.cleanupTsState = getRuntimeContext().getState(cleanupTsStateDescriptor); }
Example #18
Source File: SlidingWindowCheckMapper.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) { ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor = new ValueStateDescriptor<>("eventsSeenSoFar", new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO))); eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor); ValueStateDescriptor<Long> lastSequenceNumberDescriptor = new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO); lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor); }
Example #19
Source File: RowTimeRowsBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRowData(); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>( "lastTriggeringTsState", Types.LONG); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); ValueStateDescriptor<Long> dataCountStateDescriptor = new ValueStateDescriptor<Long>( "processedCountState", Types.LONG); counterState = getRuntimeContext().getState(dataCountStateDescriptor); RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes); ValueStateDescriptor<RowData> accStateDesc = new ValueStateDescriptor<RowData>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes); ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType); MapStateDescriptor<Long, List<RowData>> inputStateDesc = new MapStateDescriptor<Long, List<RowData>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); initCleanupTimeState("RowTimeBoundedRowsOverCleanupTime"); }
Example #20
Source File: AppendOnlyTopNFunction.java From flink with Apache License 2.0 | 5 votes |
public void open(Configuration parameters) throws Exception { super.open(parameters); int lruCacheSize = Math.max(1, (int) (cacheSize / getDefaultTopNSize())); kvSortedMap = new LRUMap<>(lruCacheSize); LOG.info("Top{} operator is using LRU caches key-size: {}", getDefaultTopNSize(), lruCacheSize); ListTypeInfo<RowData> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<RowData, List<RowData>> mapStateDescriptor = new MapStateDescriptor<>( "data-state-with-append", sortKeyType, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); // metrics registerMetric(kvSortedMap.size() * getDefaultTopNSize()); }
Example #21
Source File: ProcTimeRowsBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRowData(); // input element are all binary row as they are came from network RowDataTypeInfo inputType = new RowDataTypeInfo(inputFieldTypes); // We keep the elements received in a Map state keyed // by the ingestion time in the operator. // we also keep counter of processed elements // and timestamp of oldest element ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType); MapStateDescriptor<Long, List<RowData>> mapStateDescriptor = new MapStateDescriptor<Long, List<RowData>>( "inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo); inputState = getRuntimeContext().getMapState(mapStateDescriptor); RowDataTypeInfo accTypeInfo = new RowDataTypeInfo(accTypes); ValueStateDescriptor<RowData> stateDescriptor = new ValueStateDescriptor<RowData>("accState", accTypeInfo); accState = getRuntimeContext().getState(stateDescriptor); ValueStateDescriptor<Long> processedCountDescriptor = new ValueStateDescriptor<Long>( "processedCountState", Types.LONG); counterState = getRuntimeContext().getState(processedCountDescriptor); ValueStateDescriptor<Long> smallestTimestampDescriptor = new ValueStateDescriptor<Long>( "smallestTSState", Types.LONG); smallestTsState = getRuntimeContext().getState(smallestTimestampDescriptor); initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime"); }
Example #22
Source File: TimeIntervalJoin.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { LOGGER.debug("Instantiating JoinFunction: {} \n\n Code:\n{}", genJoinFunc.getClassName(), genJoinFunc.getCode()); joinFunction = genJoinFunc.newInstance(getRuntimeContext().getUserCodeClassLoader()); genJoinFunc = null; joinCollector = new EmitAwareCollector(); // Initialize the data caches. ListTypeInfo<Tuple2<RowData, Boolean>> leftRowListTypeInfo = new ListTypeInfo<>( new TupleTypeInfo<>(leftType, BasicTypeInfo.BOOLEAN_TYPE_INFO)); MapStateDescriptor<Long, List<Tuple2<RowData, Boolean>>> leftMapStateDescriptor = new MapStateDescriptor<>( "IntervalJoinLeftCache", BasicTypeInfo.LONG_TYPE_INFO, leftRowListTypeInfo); leftCache = getRuntimeContext().getMapState(leftMapStateDescriptor); ListTypeInfo<Tuple2<RowData, Boolean>> rightRowListTypeInfo = new ListTypeInfo<>( new TupleTypeInfo<>(rightType, BasicTypeInfo.BOOLEAN_TYPE_INFO)); MapStateDescriptor<Long, List<Tuple2<RowData, Boolean>>> rightMapStateDescriptor = new MapStateDescriptor<>( "IntervalJoinRightCache", BasicTypeInfo.LONG_TYPE_INFO, rightRowListTypeInfo); rightCache = getRuntimeContext().getMapState(rightMapStateDescriptor); // Initialize the timer states. ValueStateDescriptor<Long> leftValueStateDescriptor = new ValueStateDescriptor<>( "IntervalJoinLeftTimerState", Long.class); leftTimerState = getRuntimeContext().getState(leftValueStateDescriptor); ValueStateDescriptor<Long> rightValueStateDescriptor = new ValueStateDescriptor<>( "IntervalJoinRightTimerState", Long.class); rightTimerState = getRuntimeContext().getState(rightValueStateDescriptor); paddingUtil = new OuterJoinPaddingUtil(leftType.getArity(), rightType.getArity()); }
Example #23
Source File: SlidingWindowCheckMapper.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) { ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor = new ValueStateDescriptor<>("eventsSeenSoFar", new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO))); eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor); ValueStateDescriptor<Long> lastSequenceNumberDescriptor = new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO); lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor); }
Example #24
Source File: AppendOnlyTopNFunction.java From flink with Apache License 2.0 | 5 votes |
public void open(Configuration parameters) throws Exception { super.open(parameters); int lruCacheSize = Math.max(1, (int) (cacheSize / getDefaultTopNSize())); kvSortedMap = new LRUMap<>(lruCacheSize); LOG.info("Top{} operator is using LRU caches key-size: {}", getDefaultTopNSize(), lruCacheSize); ListTypeInfo<BaseRow> valueTypeInfo = new ListTypeInfo<>(inputRowType); MapStateDescriptor<BaseRow, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<>( "data-state-with-append", sortKeyType, valueTypeInfo); dataState = getRuntimeContext().getMapState(mapStateDescriptor); // metrics registerMetric(kvSortedMap.size() * getDefaultTopNSize()); }
Example #25
Source File: RowTimeRangeBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRow(); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>( "lastTriggeringTsState", Types.LONG); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes); ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes); ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType); MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); initCleanupTimeState("RowTimeBoundedRangeOverCleanupTime"); }
Example #26
Source File: RowTimeRowsBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRow(); ValueStateDescriptor<Long> lastTriggeringTsDescriptor = new ValueStateDescriptor<Long>( "lastTriggeringTsState", Types.LONG); lastTriggeringTsState = getRuntimeContext().getState(lastTriggeringTsDescriptor); ValueStateDescriptor<Long> dataCountStateDescriptor = new ValueStateDescriptor<Long>( "processedCountState", Types.LONG); counterState = getRuntimeContext().getState(dataCountStateDescriptor); BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes); ValueStateDescriptor<BaseRow> accStateDesc = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo); accState = getRuntimeContext().getState(accStateDesc); // input element are all binary row as they are came from network BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes); ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType); MapStateDescriptor<Long, List<BaseRow>> inputStateDesc = new MapStateDescriptor<Long, List<BaseRow>>( "inputState", Types.LONG, rowListTypeInfo); inputState = getRuntimeContext().getMapState(inputStateDesc); initCleanupTimeState("RowTimeBoundedRowsOverCleanupTime"); }
Example #27
Source File: ProcTimeRowsBoundedPrecedingFunction.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { function = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader()); function.open(new PerKeyStateDataViewStore(getRuntimeContext())); output = new JoinedRow(); // input element are all binary row as they are came from network BaseRowTypeInfo inputType = new BaseRowTypeInfo(inputFieldTypes); // We keep the elements received in a Map state keyed // by the ingestion time in the operator. // we also keep counter of processed elements // and timestamp of oldest element ListTypeInfo<BaseRow> rowListTypeInfo = new ListTypeInfo<BaseRow>(inputType); MapStateDescriptor<Long, List<BaseRow>> mapStateDescriptor = new MapStateDescriptor<Long, List<BaseRow>>( "inputState", BasicTypeInfo.LONG_TYPE_INFO, rowListTypeInfo); inputState = getRuntimeContext().getMapState(mapStateDescriptor); BaseRowTypeInfo accTypeInfo = new BaseRowTypeInfo(accTypes); ValueStateDescriptor<BaseRow> stateDescriptor = new ValueStateDescriptor<BaseRow>("accState", accTypeInfo); accState = getRuntimeContext().getState(stateDescriptor); ValueStateDescriptor<Long> processedCountDescriptor = new ValueStateDescriptor<Long>( "processedCountState", Types.LONG); counterState = getRuntimeContext().getState(processedCountDescriptor); ValueStateDescriptor<Long> smallestTimestampDescriptor = new ValueStateDescriptor<Long>( "smallestTSState", Types.LONG); smallestTsState = getRuntimeContext().getState(smallestTimestampDescriptor); initCleanupTimeState("ProcTimeBoundedRowsOverCleanupTime"); }
Example #28
Source File: TimeBoundedStreamJoin.java From flink with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) throws Exception { LOGGER.debug("Instantiating JoinFunction: {} \n\n Code:\n{}", genJoinFunc.getClassName(), genJoinFunc.getCode()); joinFunction = genJoinFunc.newInstance(getRuntimeContext().getUserCodeClassLoader()); genJoinFunc = null; joinCollector = new EmitAwareCollector(); // Initialize the data caches. ListTypeInfo<Tuple2<BaseRow, Boolean>> leftRowListTypeInfo = new ListTypeInfo<>( new TupleTypeInfo<>(leftType, BasicTypeInfo.BOOLEAN_TYPE_INFO)); MapStateDescriptor<Long, List<Tuple2<BaseRow, Boolean>>> leftMapStateDescriptor = new MapStateDescriptor<>( "WindowJoinLeftCache", BasicTypeInfo.LONG_TYPE_INFO, leftRowListTypeInfo); leftCache = getRuntimeContext().getMapState(leftMapStateDescriptor); ListTypeInfo<Tuple2<BaseRow, Boolean>> rightRowListTypeInfo = new ListTypeInfo<>( new TupleTypeInfo<>(rightType, BasicTypeInfo.BOOLEAN_TYPE_INFO)); MapStateDescriptor<Long, List<Tuple2<BaseRow, Boolean>>> rightMapStateDescriptor = new MapStateDescriptor<>( "WindowJoinRightCache", BasicTypeInfo.LONG_TYPE_INFO, rightRowListTypeInfo); rightCache = getRuntimeContext().getMapState(rightMapStateDescriptor); // Initialize the timer states. ValueStateDescriptor<Long> leftValueStateDescriptor = new ValueStateDescriptor<>( "WindowJoinLeftTimerState", Long.class); leftTimerState = getRuntimeContext().getState(leftValueStateDescriptor); ValueStateDescriptor<Long> rightValueStateDescriptor = new ValueStateDescriptor<>( "WindowJoinRightTimerState", Long.class); rightTimerState = getRuntimeContext().getState(rightValueStateDescriptor); paddingUtil = new OuterJoinPaddingUtil(leftType.getArity(), rightType.getArity()); }
Example #29
Source File: SlidingWindowCheckMapper.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public void open(Configuration parameters) { ValueStateDescriptor<List<Tuple2<Event, Integer>>> previousWindowDescriptor = new ValueStateDescriptor<>("eventsSeenSoFar", new ListTypeInfo<>(new TupleTypeInfo<>(TypeInformation.of(Event.class), BasicTypeInfo.INT_TYPE_INFO))); eventsSeenSoFar = getRuntimeContext().getState(previousWindowDescriptor); ValueStateDescriptor<Long> lastSequenceNumberDescriptor = new ValueStateDescriptor<>("lastSequenceNumber", BasicTypeInfo.LONG_TYPE_INFO); lastSequenceNumber = getRuntimeContext().getState(lastSequenceNumberDescriptor); }
Example #30
Source File: ListStateListReader.java From bravo with Apache License 2.0 | 4 votes |
@SuppressWarnings({ "rawtypes", "unchecked" }) public ListStateListReader(String stateName, TypeInformation<K> outKeyType, TypeInformation<V> outValueType) { super(stateName, outKeyType, outValueType, new TupleTypeInfo(Tuple2.class, outKeyType, new ListTypeInfo<>(outValueType))); }