Java Code Examples for org.apache.distributedlog.api.DistributedLogManager#getAppendOnlyStreamWriter()

@Test(timeout = 60000)
public void testPositionDoesntUpdateBeforeWriteCompletion() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();

    // Long flush time, but we don't wait for it.
    conf.setPeriodicFlushFrequencyMilliSeconds(100 * 1000);
    conf.setImmediateFlushEnabled(false);
    conf.setOutputBufferSize(1024 * 1024);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();

    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    assertEquals(0, writer.position());

    // Much much less than the flush time, small enough not to slow down tests too much, just
    // gives a little more confidence.
    Thread.sleep(500);
    CompletableFuture<DLSN> dlsnFuture = writer.write(byteStream);
    assertEquals(0, writer.position());

    writer.close();
    dlmwriter.close();
}
 

@Test(timeout = 60000)
public void testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(1 * 1000);
    conf.setImmediateFlushEnabled(false);
    conf.setOutputBufferSize(1024 * 1024);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();

    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    assertEquals(0, writer.position());

    Utils.ioResult(writer.write(byteStream));
    Thread.sleep(100); // let WriteCompleteListener have time to run
    assertEquals(33, writer.position());

    writer.close();
    dlmwriter.close();
}
 

@Test(timeout = 60000)
public void testBasicReadAndWriteBehavior() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes();

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 11).getBytes());
    writer.write(DLMTestUtil.repeatString("abc", 40).getBytes());
    writer.force(false);
    writer.close();
    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();

    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, 23);
    assertEquals(23, read);
    read = reader.read(bytesIn, 23, 31);
    assertEquals(read, 31);
    byte[] bytesInTemp = new byte[byteStream.length];
    read = reader.read(bytesInTemp, 0, byteStream.length);
    assertEquals(read, byteStream.length - 23 - 31);
    read = new ByteArrayInputStream(bytesInTemp).read(bytesIn, 23 + 31, byteStream.length - 23 - 31);
    assertEquals(read, byteStream.length - 23 - 31);
    assertArrayEquals(bytesIn, byteStream);
    reader.close();
    dlmreader.close();
    dlmwrite.close();
}
 

@Test(timeout = 60000)
public void testWriteFutureDoesNotCompleteUntilWritePersisted() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE);
    conf.setImmediateFlushEnabled(false);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes();

    // Can't reliably test the future is not completed until fsync is called, since writer.force may just
    // happen very quickly. But we can test that the mechanics of the future write and api are basically
    // correct.
    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    CompletableFuture<DLSN> dlsnFuture = writer.write(DLMTestUtil.repeatString("abc", 11).getBytes());

    // The real problem is the fsync completes before writes are submitted, so it never takes effect.
    Thread.sleep(1000);
    assertFalse(dlsnFuture.isDone());
    writer.force(false);
    // Must not throw.
    Utils.ioResult(dlsnFuture, 5, TimeUnit.SECONDS);
    writer.close();
    dlmwriter.close();

    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();
    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, 31);
    assertEquals(31, read);
    reader.close();
    dlmreader.close();
}
 

@Test(timeout = 60000)
public void testPositionUpdatesOnlyAfterWriteCompletion() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(10 * 1000);
    conf.setImmediateFlushEnabled(false);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();

    // Can't reliably test the future is not completed until fsync is called, since writer.force may just
    // happen very quickly. But we can test that the mechanics of the future write and api are basically
    // correct.
    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    CompletableFuture<DLSN> dlsnFuture = writer.write(byteStream);
    Thread.sleep(100);

    // Write hasn't been persisted, position better not be updated.
    assertFalse(dlsnFuture.isDone());
    assertEquals(0, writer.position());
    writer.force(false);
    // Position guaranteed to be accurate after writer.force().
    assertEquals(byteStream.length, writer.position());

    // Close writer.
    writer.close();
    dlmwriter.close();

    // Make sure we can read it.
    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();
    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, byteStream.length);
    assertEquals(byteStream.length, read);
    assertEquals(byteStream.length, reader.position());
    reader.close();
    dlmreader.close();
}
 

public void skipForwardThenSkipBack(String name, DistributedLogConfiguration conf) throws Exception {
    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 5).getBytes());
    writer.write(DLMTestUtil.repeatString("abc", 5).getBytes());
    writer.write(DLMTestUtil.repeatString("def", 5).getBytes());
    writer.write(DLMTestUtil.repeatString("def", 5).getBytes());
    writer.write(DLMTestUtil.repeatString("ghi", 5).getBytes());
    writer.write(DLMTestUtil.repeatString("ghi", 5).getBytes());
    writer.force(false);
    writer.close();

    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();
    byte[] bytesIn = new byte[30];

    byte[] bytes1 = DLMTestUtil.repeatString("abc", 10).getBytes();
    byte[] bytes2 = DLMTestUtil.repeatString("def", 10).getBytes();
    byte[] bytes3 = DLMTestUtil.repeatString("ghi", 10).getBytes();

    int read = reader.read(bytesIn, 0, 30);
    assertEquals(30, read);
    assertTrue(Arrays.equals(bytes1, bytesIn));

    reader.skipTo(60);
    read = reader.read(bytesIn, 0, 30);
    assertEquals(30, read);
    assertTrue(Arrays.equals(bytes3, bytesIn));

    reader.skipTo(30);
    read = reader.read(bytesIn, 0, 30);
    assertEquals(30, read);
    assertTrue(Arrays.equals(bytes2, bytesIn));
}
 

@Test(timeout = 60000)
public void testSkipToreturnsFalseIfPositionDoesNotExistYetForUnSealedStream() throws Exception {
    String name = testNames.getMethodName();

    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 5).getBytes());
    writer.close();

    final AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();
    byte[] bytesIn = new byte[9];

    int read = reader.read(bytesIn, 0, 9);
    assertEquals(9, read);
    assertTrue(Arrays.equals(DLMTestUtil.repeatString("abc", 3).getBytes(), bytesIn));

    assertFalse(reader.skipTo(16));
    assertFalse(reader.skipTo(16));

    AppendOnlyStreamWriter writer2 = dlmwrite.getAppendOnlyStreamWriter();
    writer2.write(DLMTestUtil.repeatString("abc", 5).getBytes());
    writer2.close();

    assertTrue(reader.skipTo(16));

    byte[] bytesIn2 = new byte[5];
    read = reader.read(bytesIn2, 0, 5);
    assertEquals(5, read);
    assertTrue(Arrays.equals("bcabc".getBytes(), bytesIn2));
}
 

@Test(timeout = 60000)
public void testSkipToForNoPositionChange() throws Exception {
    String name = testNames.getMethodName();

    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 5).getBytes());
    writer.close();

    final AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();

    assertTrue(reader.skipTo(0));

    byte[] bytesIn = new byte[4];
    int read = reader.read(bytesIn, 0, 4);
    assertEquals(4, read);
    assertEquals(new String("abca"), new String(bytesIn));

    assertTrue(reader.skipTo(reader.position()));

    assertTrue(reader.skipTo(1));

    read = reader.read(bytesIn, 0, 4);
    assertEquals(4, read);
    assertEquals(new String("bcab"), new String(bytesIn));
}
 

protected OutputStream openOutputStream(String path) throws IOException {
  DistributedLogManager dlm = namespace.openLog(path);
  AppendOnlyStreamWriter writer = dlm.getAppendOnlyStreamWriter();
  return new DLOutputStream(dlm, writer);
}
 

protected OutputStream openOutputStream(String pkgName) throws IOException {
  DistributedLogManager dlm = namespace.openLog(pkgName);
  AppendOnlyStreamWriter writer = dlm.getAppendOnlyStreamWriter();
  return new DLOutputStream(dlm, writer);
}
 

private static OutputStream openOutputStream(Namespace namespace, String pkgName)
    throws IOException {
  DistributedLogManager dlm = namespace.openLog(pkgName);
  AppendOnlyStreamWriter writer = dlm.getAppendOnlyStreamWriter();
  return new DLOutputStream(dlm, writer);
}