org.apache.flink.runtime.state.filesystem.FileStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.filesystem.FileStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SavepointV2Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
Example #2
Source File: SavepointV2Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
Example #3
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
Example #4
Source File: MetadataSerializer.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
/**
 * 解析 operatorSubtaskState 的 ManagedKeyedState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedKeyedState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 KeyedState
    for (KeyedStateHandle keyedStateHandle : operatorSubtaskState.getManagedKeyedState()) {
        // 本案例针对 Flink RocksDB 的增量 Checkpoint 引发的问题，
        // 因此仅处理 IncrementalRemoteKeyedStateHandle
        if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
            // 获取 RocksDB 的 sharedState
            Map<StateHandleID, StreamStateHandle> sharedState =
                    ((IncrementalRemoteKeyedStateHandle) keyedStateHandle).getSharedState();
            // 遍历 sharedState 中所有的 sst 文件，key 为 sst 文件名，value 为对应的 hdfs 文件 Handle
            for (Map.Entry<StateHandleID, StreamStateHandle> entry : sharedState.entrySet()) {
                // 打印 sst 文件名
                System.out.println("sstable 文件名：" + entry.getKey());
                if (entry.getValue() instanceof FileStateHandle) {
                    Path filePath = ((FileStateHandle) entry.getValue()).getFilePath();
                    // 打印 sst 文件对应的 hdfs 文件位置
                    System.out.println("sstable 文件对应的 hdfs 位置：" + filePath.getPath());
                }
            }
        }
    }
}
Example #5
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
Example #6
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
Example #7
Source File: SavepointV1Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
Example #8
Source File: MetadataSerializer.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
/**
 * 解析 operatorSubtaskState 的 ManagedKeyedState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedKeyedState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 KeyedState
    for (KeyedStateHandle keyedStateHandle : operatorSubtaskState.getManagedKeyedState()) {
        // 本案例针对 Flink RocksDB 的增量 Checkpoint 引发的问题，
        // 因此仅处理 IncrementalRemoteKeyedStateHandle
        if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
            // 获取 RocksDB 的 sharedState
            Map<StateHandleID, StreamStateHandle> sharedState =
                    ((IncrementalRemoteKeyedStateHandle) keyedStateHandle).getSharedState();
            // 遍历 sharedState 中所有的 sst 文件，key 为 sst 文件名，value 为对应的 hdfs 文件 Handle
            for (Map.Entry<StateHandleID, StreamStateHandle> entry : sharedState.entrySet()) {
                // 打印 sst 文件名
                System.out.println("sstable 文件名：" + entry.getKey());
                if (entry.getValue() instanceof FileStateHandle) {
                    Path filePath = ((FileStateHandle) entry.getValue()).getFilePath();
                    // 打印 sst 文件对应的 hdfs 文件位置
                    System.out.println("sstable 文件对应的 hdfs 位置：" + filePath.getPath());
                }
            }
        }
    }
}
Example #9
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());

	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
Example #10
Source File: MetadataSerializer.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
/**
 * 解析 operatorSubtaskState 的 ManagedOperatorState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedOperatorState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 OperatorState
    for (OperatorStateHandle operatorStateHandle : operatorSubtaskState.getManagedOperatorState()) {
        StreamStateHandle delegateStateHandle = operatorStateHandle.getDelegateStateHandle();
        if (delegateStateHandle instanceof FileStateHandle) {
            Path filePath = ((FileStateHandle) delegateStateHandle).getFilePath();
            System.out.println(filePath.getPath());
        }
    }
}
Example #11
Source File: SavepointV1Serializer.java    From flink with Apache License 2.0 5 votes vote down vote up 
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());

	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
Example #12
Source File: MetadataSerializer.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
/**
 * 解析 operatorSubtaskState 的 ManagedOperatorState
 *
 * @param operatorSubtaskState operatorSubtaskState
 */
private static void parseManagedOperatorState(OperatorSubtaskState operatorSubtaskState) {
    // 遍历当前 subtask 的 OperatorState
    for (OperatorStateHandle operatorStateHandle : operatorSubtaskState.getManagedOperatorState()) {
        StreamStateHandle delegateStateHandle = operatorStateHandle.getDelegateStateHandle();
        if (delegateStateHandle instanceof FileStateHandle) {
            Path filePath = ((FileStateHandle) delegateStateHandle).getFilePath();
            System.out.println(filePath.getPath());
        }
    }
}
Example #13
Source File: MetadataV2V3SerializerBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
static void serializeStreamStateHandle(StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof RelativeFileStateHandle) {
		dos.writeByte(RELATIVE_STREAM_STATE_HANDLE);
		RelativeFileStateHandle relativeFileStateHandle = (RelativeFileStateHandle) stateHandle;
		dos.writeUTF(relativeFileStateHandle.getRelativePath());
		dos.writeLong(relativeFileStateHandle.getStateSize());
	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
Example #14
Source File: RetrievableStreamStateHandle.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}
Example #15
Source File: RetrievableStreamStateHandle.java    From flink with Apache License 2.0 4 votes vote down vote up 
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}
Example #16
Source File: RetrievableStreamStateHandle.java    From flink with Apache License 2.0 4 votes vote down vote up 
public RetrievableStreamStateHandle(Path filePath, long stateSize) {
	Preconditions.checkNotNull(filePath);
	this.wrappedStreamStateHandle = new FileStateHandle(filePath, stateSize);
}