Java Code Examples for com.streamsets.pipeline.api.FileRef#createInputStream()

The following examples show how to use com.streamsets.pipeline.api.FileRef#createInputStream() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FileRefUtil.java From datacollector with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
public static  <T extends AutoCloseable> T getReadableStream(
    ProtoConfigurableEntity.Context context,
    FileRef fileRef,
    Class<T> streamClass,
    boolean includeChecksumInTheEvents,
    ChecksumAlgorithm checksumAlgorithm,
    StreamCloseEventHandler<?> streamCloseEventHandler
) throws IOException {
  T stream = fileRef.createInputStream(context, streamClass);
  if (includeChecksumInTheEvents) {
    Utils.checkArgument(
        FileRefStreamCloseEventHandler.class.isAssignableFrom(streamCloseEventHandler.getClass()),
        "Stream Close Event handler should be of type " + FileRefStreamCloseEventHandler.class.getCanonicalName()
    );
    stream = (T) new ChecksumCalculatingWrapperStream(stream, checksumAlgorithm.getHashType(), streamCloseEventHandler);
  }
  return stream;
}

Example 2

Source File: TestMetricsEnabledWrapperStream.java From datacollector with Apache License 2.0

6 votes

@Test
public void testIOReadParameterized() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = fileRef.createInputStream(context, InputStream.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b, 0, b.length)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}

Example 3

Source File: TestMetricsEnabledWrapperStream.java From datacollector with Apache License 2.0

6 votes

@Test
public void testNIOReadWithDirectBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocateDirect(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}

Example 4

Source File: TestMetricsEnabledWrapperStream.java From datacollector with Apache License 2.0

6 votes

@Test
public void testNIOReadWithHeapByteBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocate(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}

Example 5

Source File: TestWholeFileDataParser.java From datacollector with Apache License 2.0

6 votes

@Test
public void testParse() throws Exception {
  DataParserFactory factory = new DataParserFactoryBuilder(context, DataParserFormat.WHOLE_FILE)
      .setMaxDataLen(1000)
      .build();
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  try (DataParser parser = factory.getParser(
      "id",
      metadata,
      FileRefTestUtil.getLocalFileRef(testDir, false, null, null)
  )) {
    Assert.assertEquals("0", parser.getOffset());
    Record record = parser.parse();
    Assert.assertNotNull(record);
    Assert.assertEquals("-1", parser.getOffset());
    Assert.assertTrue(record.has("/fileInfo"));
    Assert.assertTrue(record.has("/fileRef"));
    FileRef fileRef = record.get("/fileRef").getValueAsFileRef();
    Assert.assertTrue(fileRef instanceof LocalFileRef);
    InputStream is = fileRef.createInputStream(context, InputStream.class);
    byte[] b = new byte[FileRefTestUtil.TEXT.getBytes().length];
    int bytesRead = is.read(b);
    Assert.assertEquals(FileRefTestUtil.TEXT.getBytes().length, bytesRead);
    Assert.assertArrayEquals(FileRefTestUtil.TEXT.getBytes(StandardCharsets.UTF_8), b);
  }
}

Example 6

Source File: TestMetricsEnabledWrapperStream.java From datacollector with Apache License 2.0

5 votes

@Test
public void testIOReadParameterLess() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = fileRef.createInputStream(context, InputStream.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    while(is.read() != -1) {
      remainingFileSize--;
      checkStateDuringReads(fileSize, remainingFileSize, is);
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}

Example 7

Source File: TestMetricsEnabledWrapperStream.java From datacollector with Apache License 2.0

5 votes

@Test
public void testIOReadsMixed() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = fileRef.createInputStream(context, InputStream.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    while ((bytesRead= FileRefTestUtil.randomReadMethodsWithInputStream(is)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}

Example 8