java.nio.channels.FileChannel.MapMode Java Examples

public StoreCheckpoint(final String scpPath) throws IOException {
    File file = new File(scpPath);
    MappedFile.ensureDirOK(file.getParent());
    boolean fileExists = file.exists();

    this.randomAccessFile = new RandomAccessFile(file, "rw");
    this.fileChannel = this.randomAccessFile.getChannel();
    this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MappedFile.OS_PAGE_SIZE);

    if (fileExists) {
        log.info("store checkpoint file exists, " + scpPath);
        this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0);
        this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8);
        this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16);

        log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp));
        log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp));
        log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp));
    } else {
        log.info("store checkpoint file not exists, " + scpPath);
    }
}
 

private boolean checkVersion(FileChannel channel) throws IOException
{
    if (channel.size() > 0)
    {
        channel.position(0);
        ByteBuffer buffer;

        if (useNIOMemoryMapping)
        {
            MappedByteBuffer mbb = channel.map(MapMode.READ_ONLY, 0, 8);
            mbb.load();
            buffer = mbb;
        }
        else
        {
            buffer = ByteBuffer.wrap(new byte[8]);
            channel.read(buffer);
            ((Buffer) buffer).position(0);
        }

        ((Buffer) buffer).position(0);
        long onDiskVersion = buffer.getLong();
        return (version == onDiskVersion);
    }
    return (version == 0);
}
 

private boolean create(String path) throws IOException {
    File file = new File(path);
    if (file.exists() == false) {
        if (file.createNewFile() == false) {
            return false;
        }
        RandomAccessFile raFile = new RandomAccessFile(file, "rwd");
        FileChannel fc = raFile.getChannel();
        MappedByteBuffer mappedByteBuffer = fc.map(MapMode.READ_WRITE, 0, this.fileLimitLength);
        mappedByteBuffer.put(LogEntity.MAGIC.getBytes());
        mappedByteBuffer.putInt(1);// 8 version
        mappedByteBuffer.putInt(-1);// 12next fileindex
        mappedByteBuffer.putInt(-2);// 16
        mappedByteBuffer.force();
        MappedByteBufferUtil.clean(mappedByteBuffer);
        fc.close();
        raFile.close();
        return true;
    } else {
        return false;
    }
}
 

@Override
public Closeable[] initFileWriter(ChunkInfo chunkInfo) throws Exception {
  Closeable[] fileChannels;
  FileChannel fileChannel = new RandomAccessFile(
      getHttpDownInfo().getTaskInfo().buildTaskFilePath(), "rw")
      .getChannel();
  if (getHttpDownInfo().getTaskInfo().getConnections() > 1) {
    LargeMappedByteBuffer mappedBuffer = new LargeMappedByteBuffer(fileChannel,
        MapMode.READ_WRITE, chunkInfo.getNowStartPosition(),
        chunkInfo.getEndPosition() - chunkInfo.getNowStartPosition() + 1);
    fileChannels = new Closeable[]{fileChannel, mappedBuffer};
  } else {
    fileChannels = new Closeable[]{fileChannel};
  }
  setAttr(chunkInfo, ATTR_FILE_CHANNELS, fileChannels);
  return fileChannels;
}
 

/**
 * Load the block.
 *
 * mmap and mlock the block, and then verify its checksum.
 *
 * @param length         The current length of the block.
 * @param blockIn        The block input stream.  Should be positioned at the
 *                       start.  The caller must close this.
 * @param metaIn         The meta file input stream.  Should be positioned at
 *                       the start.  The caller must close this.
 * @param blockFileName  The block file name, for logging purposes.
 *
 * @return               The Mappable block.
 */
public static MappableBlock load(long length,
    FileInputStream blockIn, FileInputStream metaIn,
    String blockFileName) throws IOException {
  MappableBlock mappableBlock = null;
  MappedByteBuffer mmap = null;
  FileChannel blockChannel = null;
  try {
    blockChannel = blockIn.getChannel();
    if (blockChannel == null) {
      throw new IOException("Block InputStream has no FileChannel.");
    }
    mmap = blockChannel.map(MapMode.READ_ONLY, 0, length);
    NativeIO.POSIX.getCacheManipulator().mlock(blockFileName, mmap, length);
    verifyChecksum(length, metaIn, blockChannel, blockFileName);
    mappableBlock = new MappableBlock(mmap, length);
  } finally {
    IOUtils.closeQuietly(blockChannel);
    if (mappableBlock == null) {
      if (mmap != null) {
        NativeIO.POSIX.munmap(mmap); // unmapping also unlocks
      }
    }
  }
  return mappableBlock;
}
 

@Override
public synchronized MappedPage allocate(int size, boolean thief, boolean victim, OffHeapStorageArea owner) {
  Long address = allocateRegion(size);
  if (address == null) {
    return null;
  }

  try {
    MappedByteBuffer buffer = channel.map(MapMode.READ_WRITE, address, size);
    MappedPage page = new MappedPage(buffer);
    pages.put(page, address);
    return page;
  } catch (IOException e) {
    freeRegion(address);
    LOGGER.warn("Mapping a new file section failed", e);
    return null;
  }
}
 

/**
 * on posix systems: allocates disk-backed bytebuffer and immediately unlinks the file
 * on others: simply returns a direct bytebuffer
 */
public static ByteBuffer allocate(int size) {
	if(MAP_AND_UNLINK_SUPPORTED) {
		try {
			Path p = Files.createTempFile("anon-mapping", ".tmp");
			ByteBuffer mapped;
			FileChannel chan = FileChannel.open(p, StandardOpenOption.READ, StandardOpenOption.WRITE);
			chan.position(size);
			chan.write(ByteBuffer.allocate(1));
			mapped = chan.map(MapMode.READ_WRITE, 0, size);
			chan.close();
			Files.delete(p);
			return mapped;
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	return ByteBuffer.allocateDirect(size);
}
 

public CrailBuffer allocateRegion() throws IOException {
	if (currentRegion >= allocationCount){
		return null;
	}

	String path = directory + "/" + currentRegion++;
	RandomAccessFile randomFile = new RandomAccessFile(path, "rw");
	randomFile.setLength(CrailConstants.REGION_SIZE);
	FileChannel channel = randomFile.getChannel();
	MappedByteBuffer _mappedBuffer = channel.map(MapMode.READ_WRITE, 0,
			CrailConstants.REGION_SIZE);
	CrailBuffer mappedBuffer = OffHeapBuffer.wrap(_mappedBuffer);
	randomFile.close();
	channel.close();

	CrailBuffer firstBuffer = slice(mappedBuffer, 0);

	for (int j = 1; j < bufferCount; j++) {
		int position = j * CrailConstants.BUFFER_SIZE;
		CrailBuffer sliceBuffer = slice(mappedBuffer, position);
		this.putBufferInternal(sliceBuffer);
	}
	mappedBuffer.clear();

	return firstBuffer;
}
 

/**
 * @tests java.nio.channels.FileChannel#map(MapMode,long,long)
 */
public void test_map_ReadWrite_NonZeroPosition() throws IOException {
    // test position non-zero
    writeDataToFile(fileOfReadWriteFileChannel);
    MappedByteBuffer mapped = readWriteFileChannel.map(MapMode.READ_WRITE,
            10, CONTENT_LENGTH - 10);
    assertEquals(CONTENT_LENGTH - 10, mapped.limit());
    assertEquals(CONTENT.length() - 10, mapped.capacity());
    assertEquals(0, mapped.position());
    mapped.put(TEST_BYTES);
    ByteBuffer checkBuffer = ByteBuffer.allocate(CONTENT_LENGTH);
    readWriteFileChannel.read(checkBuffer);
    String expected = CONTENT.substring(0, 10) + "test"
            + CONTENT.substring(10 + "test".length());
    assertEquals(expected, new String(checkBuffer.array(), "iso8859-1"));
}
 

/**
 * @tests java.nio.channels.FileChannel#map(MapMode,long,long)
 */
public void test_map_ReadWrite() throws IOException {
    MappedByteBuffer mapped = null;
    writeDataToFile(fileOfReadWriteFileChannel);
    mapped = readWriteFileChannel.map(MapMode.READ_WRITE, 0, CONTENT
            .length());

    // put something will change its channel
    ByteBuffer returnByPut = mapped.put(TEST_BYTES);
    assertSame(returnByPut, mapped);
    String checkString = "test" + CONTENT.substring(4);
    ByteBuffer checkBuffer = ByteBuffer.allocate(CONTENT_LENGTH);
    mapped.force();
    readWriteFileChannel.position(0);
    readWriteFileChannel.read(checkBuffer);
    assertEquals(checkString, new String(checkBuffer.array(), "iso8859-1"));

    try {
        mapped.put(("test" + CONTENT).getBytes("iso8859-1"));
        fail("should throw BufferOverflowException.");
    } catch (BufferOverflowException ex) {
        // expected;
    }
}
 

public StoreCheckpoint(final String scpPath) throws IOException {
    File file = new File(scpPath);
    MappedFile.ensureDirOK(file.getParent());
    boolean fileExists = file.exists();

    this.randomAccessFile = new RandomAccessFile(file, "rw");
    this.fileChannel = this.randomAccessFile.getChannel();
    this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MappedFile.OS_PAGE_SIZE);

    if (fileExists) {
        log.info("store checkpoint file exists, " + scpPath);
        this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0);
        this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8);
        this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16);

        log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp));
        log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp));
        log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp));
    } else {
        log.info("store checkpoint file not exists, " + scpPath);
    }
}
 

public StoreCheckpoint(final String scpPath) throws IOException {
    File file = new File(scpPath);
    MappedFile.ensureDirOK(file.getParent());
    boolean fileExists = file.exists();

    this.randomAccessFile = new RandomAccessFile(file, "rw");
    this.fileChannel = this.randomAccessFile.getChannel();
    this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MappedFile.OS_PAGE_SIZE);

    if (fileExists) {
        log.info("store checkpoint file exists, " + scpPath);
        this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0);
        this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8);
        this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16);

        log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp));
        log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp));
        log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", "
            + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp));
    } else {
        log.info("store checkpoint file not exists, " + scpPath);
    }
}
 

private static void testExceptions(FileChannel fc) throws IOException {
    checkException(fc, null, 0L, fc.size(),
                   NullPointerException.class);

    checkException(fc, MapMode.READ_ONLY, -1L, fc.size(),
                   IllegalArgumentException.class);

    checkException(fc, null, -1L, fc.size(),
                   IllegalArgumentException.class, NullPointerException.class);

    checkException(fc, MapMode.READ_ONLY, 0L, -1L,
                   IllegalArgumentException.class);

    checkException(fc, null, 0L, -1L,
                   IllegalArgumentException.class, NullPointerException.class);

    checkException(fc, MapMode.READ_ONLY, 0L, Integer.MAX_VALUE + 1L,
                   IllegalArgumentException.class);

    checkException(fc, null, 0L, Integer.MAX_VALUE + 1L,
                   IllegalArgumentException.class, NullPointerException.class);

    checkException(fc, MapMode.READ_ONLY, Long.MAX_VALUE, 1L,
                   IllegalArgumentException.class);

    checkException(fc, null, Long.MAX_VALUE, 1L,
                   IllegalArgumentException.class, NullPointerException.class);

}
 

@Override
public void run() {
    try {
        gate.acquireUninterruptibly();
        fc.map(MapMode.READ_ONLY, 0, 1);
        throw new Exception("Map succeeded");
    } catch (IOException x) {
        System.out.println(x.getClass() + " (expected)");
    } catch (Exception unexpected) {
        this.exception = unexpected;
    }
}
 

@SuppressWarnings("resource")
public LogWriterImpl(File databaseDir, long fileNumber, boolean isManifest) throws IOException {
	Preconditions.checkArgument(fileNumber >= 1);
	this.fileNumber = fileNumber;
	// 新建manifest或普通log文件
	File file = FileUtils.newLogFile(databaseDir, fileNumber, isManifest);
	// 绑定file channel
	channel = new RandomAccessFile(file, "rw").getChannel();
	// 获取mmap
	mmap = channel.map(MapMode.READ_WRITE, 0, LOG_BLOCK_SIZE);
	// 初始化位置信息
	lastPos = 0;
}
 

private static MapMode newMapMode(String name) {
    try {
        return (MapMode) MAP_MODE_CONSTRUCTOR.invoke(name);
    } catch (Throwable e) {
        throw new InternalError(e);
    }
}
 

public static void main(String... args) throws Exception {
	String base = getBasePath();
	loadJars(base);
	try (RandomAccessFile raf = new RandomAccessFile(Config.base() + "/command.swap", "rw")) {
		FileChannel fc = raf.getChannel();
		MappedByteBuffer mbb = fc.map(MapMode.READ_WRITE, 0, 256);
		FileLock flock = null;
		flock = fc.lock();
		mbb.put("exit".getBytes());
		flock.release();
	}
}
 

@Override
public void run() {
    try {
        gate.acquireUninterruptibly();
        fc.map(MapMode.READ_ONLY, 0, 1);
        throw new Exception("Map succeeded");
    } catch (IOException x) {
        System.out.println(x.getClass() + " (expected)");
    } catch (Exception unexpected) {
        this.exception = unexpected;
    }
}
 

@Override
public StorageResource allocateResource() throws Exception {
	StorageResource resource = null;
	
	if (allocatedSize < RdmaConstants.STORAGE_RDMA_STORAGE_LIMIT){
		//mmap buffer
		int fileId = fileCount++;
		String dataFilePath = dataDirPath + "/" + fileId;
		RandomAccessFile dataFile = new RandomAccessFile(dataFilePath, "rw");
		if (!RdmaConstants.STORAGE_RDMA_PERSISTENT){
			dataFile.setLength(RdmaConstants.STORAGE_RDMA_ALLOCATION_SIZE);
		}
		FileChannel dataChannel = dataFile.getChannel();
		ByteBuffer dataBuffer = dataChannel.map(MapMode.READ_WRITE, 0, RdmaConstants.STORAGE_RDMA_ALLOCATION_SIZE);
		dataFile.close();
		dataChannel.close();

		//register buffer
		allocatedSize += dataBuffer.capacity();
		IbvMr mr = datanodeServerEndpoint.registerMemory(dataBuffer).execute().free().getMr();

		//create resource
		resource = StorageResource.createResource(mr.getAddr(), mr.getLength(), mr.getLkey());
	}
	
	return resource;
}
 

public synchronized MappedPage claimPage(long address, long size) throws IOException {
  claimRegion(address, size);
  MappedByteBuffer buffer = channel.map(MapMode.READ_WRITE, address, size);
  MappedPage page = new MappedPage(buffer);
  pages.put(page, address);
  return page;
}
 

public void force(long offset, int length) throws IOException {
    _log.debug(String.format("forcing %s offset 0x%08x length 0x%08x", file.toString(), offset, length));
    MappedByteBuffer buff = null;
    try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
        FileChannel channel = raf.getChannel();
        buff = channel.map(MapMode.READ_WRITE, offset, size);
        buff.force();
    }
}
 

/**
 * 确保mmap空间满足需求
 * @param need 需要的空间
 * @throws IOException
 */
private void applyFor(int need) throws IOException {
	// 如果mmap剩余空间已经不满足需求
	if(mmap.remaining() < need) {
		// 将page cache中的数据刷进硬盘
		mmap.force();
		// 释放mmap
		MmapReleaseUtil.clean(mmap);
		// 更新位置并且重新申请mmap
		lastPos += LOG_BLOCK_SIZE;
		mmap = channel.map(MapMode.READ_WRITE, lastPos, LOG_BLOCK_SIZE);
	}
}
 

@Test
void testMap() throws IOException {
	MappedByteBuffer mockMappedByteBuffer = mock(MappedByteBuffer.class);
	when(fc.map(any(MapMode.class), anyLong(), anyLong())).thenReturn(mockMappedByteBuffer);

	assertThat(hdfFc.map(20, 10), is(sameInstance(mockMappedByteBuffer)));
}
 

private void openMutable() throws IOException {
    if (!this.file.exists()) {
        _log.info("creating new checkpoint file {}", this.file);
    }
    try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) {
        this.ch = raf.getChannel();
        for (UberTimer timer = new UberTimer(60 * 1000); !timer.isExpired();) {
            FileLock lock = this.ch.tryLock();
            if (lock != null) {
                this.buf = this.ch.map(MapMode.READ_WRITE, 0, 512);
                this.addr = UberUtil.getAddress(buf);
                if (getServerId() == 0) {
                    Unsafe.putLong(this.addr + OFFSET_SERVER_ID, System.nanoTime());
                }
                return;
            }
            try {
                Thread.sleep(1000);
            }
            catch (InterruptedException ignored) {
            }
        }
    }
    finally {
        if (this.ch != null) {
            this.ch.close();
        }
    }
    throw new IOException("unable to acquire lock on " + file.getAbsolutePath());
}
 

/**
 * Tests zero size file mapping
 */
private static void testZero() throws Exception {
    try (FileInputStream fis = new FileInputStream(blah)) {
        FileChannel fc = fis.getChannel();
        MappedByteBuffer b = fc.map(MapMode.READ_ONLY, 0, 0);
    }
}
 

@Override
public DoubleVector get(String word) throws IOException {

  if (!contains(word)) {
    return null;
  }

  long offset = dictMap.get(word);

  // page the block in, read from it and wrap as a vector
  MappedByteBuffer buf = raf.getChannel()
      .map(MapMode.READ_ONLY, offset, size);

  return parse(buf);
}
 

private void init(long readIndex) {
    this.isEnd = false;
    this.readIndex = readIndex;
    this.next = null;
    try {
        this.af = new RandomAccessFile(file, "r");
        this.fc = af.getChannel();
        this.readBuffer = fc.map(MapMode.READ_ONLY, readIndex, mappingSize);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

private static MappedByteBuffer map(RandomAccessFile raf, MapMode mode, long size)
    throws IOException {
  Closer closer = Closer.create();
  try {
    FileChannel channel = closer.register(raf.getChannel());
    return channel.map(mode, 0, size);
  } catch (Throwable e) {
    throw closer.rethrow(e);
  } finally {
    closer.close();
  }
}
 

private void rollOverToNextBuffer() throws IOException {
	if (currentBuffer != null) {
		// we need to remember the original buffers, not any slices.
		// slices have no cleaner, which we need to trigger explicit unmapping
		currentBuffer.flip();
		fullBuffers.add(currentBuffer);
	}

	currentBuffer = file.map(MapMode.READ_WRITE, nextMappingOffset, mappingSize);
	currentBuffer.order(ByteOrder.nativeOrder());
	nextMappingOffset += mappingSize;
}
 

private static MappedByteBuffer map(RandomAccessFile raf, MapMode mode, long size) throws IOException {
  Closer closer = Closer.create();
  try {
    FileChannel channel = closer.register(raf.getChannel());
    return channel.map(mode, 0, size);
  } catch (Throwable e) {
    throw closer.rethrow(e);
  } finally {
    closer.close();
  }
}