com.streamsets.pipeline.api.FileRef Java Examples

@Override
public String convert(Object value) {
  // Shortcut if the input is already string
  if(value instanceof String) {
    return (String)value;
  }

  if(value instanceof Map || value instanceof List || value instanceof byte[]|| value instanceof FileRef) {
    throw new IllegalArgumentException(Utils.format(Errors.API_18.getMessage()));
  }
  // ZoneDatetime.toString() does not use a standard format which can be parsed.
  if (value instanceof ZonedDateTime) {
    return Utils.format((ZonedDateTime) value);
  }

  return value.toString();
}
 

/**
 * Return the Avro file input stream
 * @param record the {@link com.streamsets.pipeline.api.Record} whole file record
 */
private InputStream getAvroInputStream(Record record) throws StageException {
  try {
    FileRef fileRef = record.get(FileRefUtil.FILE_REF_FIELD_PATH).getValueAsFileRef();

    // get avro reader
    final boolean includeChecksumInTheEvents = false;

    InputStream is = FileRefUtil.getReadableStream(
        getContext(),
        fileRef,
        InputStream.class,
        includeChecksumInTheEvents,
        null,
        null
    );

    return is;
  } catch (IOException ex) {
    throw new TransformerStageCheckedException(Errors.CONVERT_07, ex.toString(), ex);
  }
}
 

@Test
public void testParse() throws Exception {
  DataParserFactory factory = new DataParserFactoryBuilder(context, DataParserFormat.WHOLE_FILE)
      .setMaxDataLen(1000)
      .build();
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  try (DataParser parser = factory.getParser(
      "id",
      metadata,
      FileRefTestUtil.getLocalFileRef(testDir, false, null, null)
  )) {
    Assert.assertEquals("0", parser.getOffset());
    Record record = parser.parse();
    Assert.assertNotNull(record);
    Assert.assertEquals("-1", parser.getOffset());
    Assert.assertTrue(record.has("/fileInfo"));
    Assert.assertTrue(record.has("/fileRef"));
    FileRef fileRef = record.get("/fileRef").getValueAsFileRef();
    Assert.assertTrue(fileRef instanceof LocalFileRef);
    InputStream is = fileRef.createInputStream(context, InputStream.class);
    byte[] b = new byte[FileRefTestUtil.TEXT.getBytes().length];
    int bytesRead = is.read(b);
    Assert.assertEquals(FileRefTestUtil.TEXT.getBytes().length, bytesRead);
    Assert.assertArrayEquals(FileRefTestUtil.TEXT.getBytes(StandardCharsets.UTF_8), b);
  }
}
 

@Test
public void testFileRefTypeError() throws Exception {
  File testDir = new File("target", UUID.randomUUID().toString());
  testDir.mkdirs();
  try {
    FileRefTestUtil.writePredefinedTextToFile(testDir);
    Stage.Context context = ContextInfoCreator.createTargetContext("i", false, OnRecordError.TO_ERROR);
    Record record = context.createRecord("id");
    Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
    FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, false, null, null);
    record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
    DataGenerator gen = new JsonCharDataGenerator(getContext(), new StringWriter(), Mode.MULTIPLE_OBJECTS);
    gen.write(record);
    Assert.fail("Json should not process FileRef field");
  } catch (DataGeneratorException e) {
    Assert.assertEquals(Errors.JSON_GENERATOR_01, e.getErrorCode());
  } finally {
    testDir.delete();
  }
}
 

@Test
public void testNIOReadWithHeapByteBuffer() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = Mockito.spy(fileRef.createInputStream(context, ReadableByteChannel.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    ByteBuffer b = ByteBuffer.allocate(10);
    int bytesRead;
    int freeSpaceInBuffer = b.remaining();
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, freeSpaceInBuffer, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
      b.compact();
      freeSpaceInBuffer = b.remaining();
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 

@Test
public void testNIOReadWithDirectBuffer() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = Mockito.spy(fileRef.createInputStream(context, ReadableByteChannel.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    ByteBuffer b = ByteBuffer.allocateDirect(10);
    int bytesRead;
    int freeSpaceInBuffer = b.remaining();
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, freeSpaceInBuffer, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
      b.compact();
      freeSpaceInBuffer = b.remaining();
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 

@Test
public void testIOReadsMixed() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    while ((bytesRead= FileRefTestUtil.randomReadMethodsWithInputStream(is)) > 0) {
      remainingFileSize -= bytesRead;
      Assert.assertTrue(isRateLimiterAcquired.get());
      Assert.assertEquals(remainingFileSize, getRemainingStreamSize(is));
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 

@Test
public void testIOReadParameterized2() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b, 0, b.length)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, b.length, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 

@Test
public void testIOReadParameterized1() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkState(is, remainingFileSize, b.length, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
    Assert.assertFalse(isRateLimiterAcquired.get());
  }
}
 

@Test
public void testIOReadParameterLess() throws Exception {
  FileRef fileRef = getLocalFileRef(testDir, RATE_LIMIT);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = Mockito.spy(fileRef.createInputStream(context, InputStream.class))) {
    AtomicInteger bytesWishToBeRead = new AtomicInteger(-1);
    AtomicBoolean isRateLimiterAcquired = new AtomicBoolean(false);
    intercept(is, bytesWishToBeRead, isRateLimiterAcquired);
    Assert.assertEquals(fileSize, getRemainingStreamSize(is));
    while(is.read() != -1) {
      remainingFileSize--;
      checkState(is, remainingFileSize, 1, bytesWishToBeRead, isRateLimiterAcquired);
      bytesWishToBeRead.set(-1);
      isRateLimiterAcquired.set(false);
    }
  }
}
 

public static FileRef getLocalFileRefWithCustomFile(
    File file,
    boolean createMetrics,
    String checksum,
    HashingUtil.HashType checksumAlgorithm
) {
  AbstractSpoolerFileRef.Builder builder = new LocalFileRef.Builder()
      .filePath(file.getAbsolutePath())
      //To force multiple reads from the file.
      .bufferSize(TEXT.getBytes().length / 2)
      .totalSizeInBytes(file.length());
  if (checksum != null) {
    builder.verifyChecksum(true)
        .checksum(checksum)
        .checksumAlgorithm(checksumAlgorithm);
  }
  if (createMetrics) {
    builder.createMetrics(true);
  } else {
    builder.createMetrics(false);
  }
  return builder.build();
}
 

@Test
public void testNIOReadWithHeapByteBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocate(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 

@Test
public void testNIOReadWithDirectBuffer() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (ReadableByteChannel is = fileRef.createInputStream(context, ReadableByteChannel.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    ByteBuffer b = ByteBuffer.allocateDirect(10);
    while((bytesRead = is.read(b)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
      b.compact();
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 

@Test
public void testIOReadParameterized() throws Exception {
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, null, null);
  long fileSize = Files.size(Paths.get(FileRefTestUtil.getSourceFilePath(testDir)));
  long remainingFileSize = fileSize;
  try (InputStream is = fileRef.createInputStream(context, InputStream.class)) {
    Assert.assertEquals(remainingFileSize, getRemainingBytes(is));
    int bytesRead;
    byte[] b = new byte[10];
    while( (bytesRead = is.read(b, 0, b.length)) > 0) {
      remainingFileSize -= bytesRead;
      checkStateDuringReads(fileSize, remainingFileSize, is);
    }
    checkStateAfterReadCompletion(fileSize, remainingFileSize, is);
  }
}
 

@Test
public void testConvertFromFilRefInvalid() {
  FileRef fileRef = new TestFileRef.ByteArrayRef("This is file ref Field".getBytes());
  testConvertFromFilRefInvalid(new IntegerTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new LongTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new FloatTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new DoubleTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new DecimalTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ByteTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new CharTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new BooleanTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ByteArrayTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new StringTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ListMapTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new ListTypeSupport(), fileRef);
  testConvertFromFilRefInvalid(new MapTypeSupport(), fileRef);
}
 

static Record getFileRefRecordForFile(Path filePath) throws Exception {
  Record fileRefRecord = RecordCreator.create();
  FileRef fileRef =
      new LocalFileRef.Builder()
          .bufferSize(1024)
          .createMetrics(false)
          .verifyChecksum(false)
          .filePath(filePath.toAbsolutePath().toString())
          .build();
  Map<String, Field> fieldMap = new HashMap<>();

  Map<String, Object> metadata = new HashMap<>(Files.readAttributes(filePath, "posix:*"));
  metadata.put("filename", filePath.getFileName());
  metadata.put("file", filePath.toString());
  metadata.put("dir", filePath.getParent().toString());
  metadata.put("permissions", "777");

  fieldMap.put(FileRefUtil.FILE_REF_FIELD_NAME, Field.create(Field.Type.FILE_REF, fileRef));
  fieldMap.put(FileRefUtil.FILE_INFO_FIELD_NAME, FileRefUtil.createFieldForMetadata(metadata));
  fileRefRecord.set(Field.create(fieldMap));

  return fileRefRecord;
}
 

@SuppressWarnings("unchecked")
public static  <T extends AutoCloseable> T getReadableStream(
    ProtoConfigurableEntity.Context context,
    FileRef fileRef,
    Class<T> streamClass,
    boolean includeChecksumInTheEvents,
    ChecksumAlgorithm checksumAlgorithm,
    StreamCloseEventHandler<?> streamCloseEventHandler
) throws IOException {
  T stream = fileRef.createInputStream(context, streamClass);
  if (includeChecksumInTheEvents) {
    Utils.checkArgument(
        FileRefStreamCloseEventHandler.class.isAssignableFrom(streamCloseEventHandler.getClass()),
        "Stream Close Event handler should be of type " + FileRefStreamCloseEventHandler.class.getCanonicalName()
    );
    stream = (T) new ChecksumCalculatingWrapperStream(stream, checksumAlgorithm.getHashType(), streamCloseEventHandler);
  }
  return stream;
}
 

@Override
public DataParser getParser(
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
) throws DataParserException {
  return new WrapperDataParser(factory.getParser(id, metadata, fileRef));
}
 

/** {@inheritDoc} */
@Override
protected void process(Record record, SingleLaneBatchMaker batchMaker) throws StageException {
  LOG.info("Input record: {}", record);

  FileRef fileRef = record.get("/fileRef").getValueAsFileRef();
  Metadata metadata;
  try {
    metadata = ImageMetadataReader.readMetadata(fileRef.createInputStream(getContext(), InputStream.class));
  } catch (ImageProcessingException | IOException e) {
    String filename = record.get("/fileInfo/filename").getValueAsString();
    LOG.info("Exception getting metadata from {}", filename, e);
    throw new OnRecordErrorException(record, Errors.SAMPLE_02, e);
  }

  for (Directory directory : metadata.getDirectories()) {
    LinkedHashMap<String, Field> listMap = new LinkedHashMap<>();

    for (Tag tag : directory.getTags()) {
      listMap.put(tag.getTagName(), Field.create(tag.getDescription()));
    }

    if (directory.hasErrors()) {
      for (String error : directory.getErrors()) {
        LOG.info("ERROR: {}", error);
      }
    }

    record.set("/" + directory.getName(), Field.createListMap(listMap));
  }

  LOG.info("Output record: {}", record);

  batchMaker.addRecord(record);
}
 

WholeFileDataParser(
    ProtoConfigurableEntity.Context context,
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
)  {
  this.id = id;
  this.context = context;
  this.metadata = metadata;
  this.fileRef = fileRef;
}
 

@Override
public DataParser getParser(
    String id,
    Map<String, Object> metadata,
    FileRef fileRef
) throws DataParserException {
  Utils.checkNotNull(fileRef, FileRefUtil.FILE_REF_FIELD_NAME);
  validateMetadata(metadata);
  return new WholeFileDataParser(
      getSettings().getContext(),
      id,
      metadata,
      fileRef
  );
}
 

private void testConvertFromFilRefInvalid(TypeSupport targetTypeSupport, FileRef fileRef) {
  try {
    new FileRefTypeSupport().convert(fileRef, targetTypeSupport);
    Assert.fail("Convert File Ref Object to " + targetTypeSupport + " should fail.");
  } catch (IllegalArgumentException e) {
    Assert.assertEquals(Utils.format(Errors.API_24.getMessage(), targetTypeSupport), e.getMessage());
  }
}
 

private Record createRecord(String checksum, HashingUtil.HashType checksumAlgorithm) throws Exception {
  Record record = context.createRecord("id");
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, true, checksum, checksumAlgorithm);
  record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
  return record;
}
 

private void getParserWithInvalidMetadata(
    FileRef fileRef,
    Map<String, Object> metadata,
    DataParserFactory dataParserFactory,
    ErrorCode errorCode
) {
  try {
    dataParserFactory.getParser("id", metadata, fileRef);
    Assert.fail("getParser should fail for missing mandatory metadata");
  } catch (DataParserException e) {
    Assert.assertEquals(errorCode, e.getErrorCode());
  }
}
 

@Test
public void testWholeFileDataParserFactoryMissingMetadata() throws Exception {
  DataParserFactory factory = new DataParserFactoryBuilder(context, DataParserFormat.WHOLE_FILE)
      .setMaxDataLen(1000)
      .build();
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadata(testDir);
  FileRef fileRef = FileRefTestUtil.getLocalFileRef(testDir, false, null, null);
  Set<String> keys = new HashSet<>(FileRefUtil.MANDATORY_METADATA_INFO);
  for (String key :keys) {
    Map<String, Object> metadataCopy = new HashMap<>(metadata);
    metadataCopy.remove(key);
    getParserWithInvalidMetadata(fileRef, metadataCopy, factory, Errors.WHOLE_FILE_PARSER_ERROR_0);
  }
}
 

@Override
public FileRef convert(Object value) {
  if (value instanceof FileRef) {
    return (FileRef)value;
  }
  throw new IllegalArgumentException(Utils.format(Errors.API_23.getMessage(), value.getClass().getName()));
}
 

public static DataParser getParser(
    WrappedFileSystem fs,
    WrappedFile file,
    DataFormat dataFormat,
    DataParserFactory parserFactory,
    String offset,
    int wholeFileMaxObjectLen,
    ELEval rateLimitElEval,
    ELVars rateLimitElVars,
    String rateLimit
) throws DataParserException, ELEvalException, IOException {
  DataParser parser;

  switch (dataFormat) {
    case WHOLE_FILE:
      FileRef fileRef = fs.getFileRefBuilder()
          .filePath(file.getAbsolutePath())
          .bufferSize(wholeFileMaxObjectLen)
          .rateLimit(FileRefUtil.evaluateAndGetRateLimit(rateLimitElEval, rateLimitElVars, rateLimit))
          .createMetrics(true)
          .totalSizeInBytes(file.getSize())
          .build();
      parser = parserFactory.getParser(file.getFileName(), file.getFileMetadata(), fileRef);
      break;
    default:
      parser = parserFactory.getParser(file.getFileName(), file.getInputStream(), offset);
  }

  return parser;
}
 

@Override
public DataParser getParser(String id, Map<String, Object> metadata, FileRef fileRef) throws DataParserException {
  try {
    return new DataParserWrapper(dataFormatConfig.getParserFactory().getParser(id, metadata, fileRef));
  } catch (com.streamsets.pipeline.lib.parser.DataParserException e) {
    throw ShimUtil.convert(e);
  }
}
 

@Override // From DataFormatParserService
public DataParser getParser(String id, Map<String, Object> metadata, FileRef fileRef) throws DataParserException {
  ClassLoader cl = serviceBean.getDefinition().getStageClassLoader();

  return LambdaUtil.privilegedWithClassLoader(
    cl,
    DataParserException.class,
    () -> new DataParserServiceWrapper(cl, ((DataFormatParserService)serviceBean.getService()).getParser(id, metadata, fileRef))
  );
}
 

private Record createRecord(File file) throws Exception {
  Record record = context.createRecord("id");
  Map<String, Object> metadata = FileRefTestUtil.getFileMetadataWithCustomFile(file);
  FileRef fileRef = FileRefTestUtil.getLocalFileRefWithCustomFile(file, true, null, null);
  record.set(FileRefUtil.getWholeFileRecordRootField(fileRef, metadata));
  return record;
}