org.apache.nifi.processor.exception.FlowFileAccessException Java Examples

@Override
public int read() throws IOException {
    try {
        final int byteRead = super.read();
        if (byteRead < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed++;
        }

        return byteRead;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 

@Override
public MockFlowFile importFrom(final Path path, final boolean keepSourceFile, final FlowFile flowFile) {
    validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;
    MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        Files.copy(path, baos);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }

    newFlowFile.setData(baos.toByteArray());
    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 

@Override
public MockFlowFile importFrom(final InputStream in, final FlowFile flowFile) {
    validateState(flowFile);
    if (in == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;

    final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);
    try {
        final byte[] data = readFully(in);
        newFlowFile.setData(data);
        return newFlowFile;
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public void exportTo(final FlowFile flowFile, final OutputStream out) {
    validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    try {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public void exportTo(final FlowFile flowFile, final Path path, final boolean append) {
    validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

private void resetWriteClaims(final boolean suppressExceptions) {
    for (final ByteCountingOutputStream out : appendableStreams.values()) {
        try {
            try {
                out.flush();
            } finally {
                out.close();
            }
        } catch (final IOException e) {
            if (!suppressExceptions) {
                throw new FlowFileAccessException("Unable to flush the output of FlowFile to the Content Repository");
            }
        }
    }
    appendableStreams.clear();
}
 

@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
    try {
        final int count = super.read(b, off, len);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 

@Override
public void exportTo(FlowFile source, final Path destination, final boolean append) {
    verifyTaskActive();
    source = validateRecordState(source);
    final StandardRepositoryRecord record = getRecord(source);
    try {
        ensureNotAppending(record.getCurrentClaim());

        claimCache.flush(record.getCurrentClaim());
        final long copyCount = context.getContentRepository().exportTo(record.getCurrentClaim(), destination, append, record.getCurrentClaimOffset(), source.getSize());
        bytesRead += copyCount;
        bytesWritten += copyCount;
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final Throwable t) {
        throw new FlowFileAccessException("Failed to export " + source + " to " + destination + " due to " + t.toString(), t);
    }
}
 

@Override
public int read() throws IOException {
    try {
        final int byteRead = super.read();
        if (byteRead < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed++;
        }

        return byteRead;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 

@Override
public void exportTo(final FlowFile source, final Path destination, final boolean append) {
    validateRecordState(source);
    final StandardRepositoryRecord record = records.get(source);
    try {
        ensureNotAppending(record.getCurrentClaim());

        claimCache.flush(record.getCurrentClaim());
        final long copyCount = context.getContentRepository().exportTo(record.getCurrentClaim(), destination, append, record.getCurrentClaimOffset(), source.getSize());
        bytesRead += copyCount;
        bytesWritten += copyCount;
    } catch (final ContentNotFoundException nfe) {
        handleContentNotFound(nfe, record);
    } catch (final Throwable t) {
        throw new FlowFileAccessException("Failed to export " + source + " to " + destination + " due to " + t.toString(), t);
    }
}
 

@Override
public int read(final byte[] b, final int off, final int len) throws IOException {
    try {
        final int count = super.read(b, off, len);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 

@Override
public int read(final byte[] b) throws IOException {
    try {
        final int count = super.read(b);
        if (count < 0) {
            ensureAllContentRead();
        } else {
            bytesConsumed += count;
        }

        return count;
    } catch (final ContentNotFoundException cnfe) {
        throw cnfe;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not read from " + flowFile, ioe);
    }
}
 

private void resetWriteClaims(final boolean suppressExceptions) {
    for (final ByteCountingOutputStream out : appendableStreams.values()) {
        try {
            try {
                out.flush();
            } finally {
                out.close();
            }
        } catch (final IOException e) {
            if (!suppressExceptions) {
                throw new FlowFileAccessException("Unable to flush the output of FlowFile to the Content Repository");
            }
        }
    }
    appendableStreams.clear();
}
 

@Override
public StatelessFlowFile importFrom(final Path path, final boolean keepSourceFile, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    if (!keepSourceFile) {
        throw new IllegalArgumentException("Not going to delete the file...");
    }
    StatelessFlowFile newFlowFile = new StatelessFlowFile((StatelessFlowFile) flowFile, this.materializeContent);
    try {
        newFlowFile.setData(Files.newInputStream(path));
    } catch (IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 

@Override
public void exportTo(FlowFile flowFile, final Path path, final boolean append) {
    flowFile = validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    StatelessFlowFile statelessFlowFile = (StatelessFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        if (statelessFlowFile.materializeContent)
            statelessFlowFile.materializeData();
        copyTo(statelessFlowFile.getDataStream(), out);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public void exportTo(FlowFile flowFile, final OutputStream out) {
    flowFile = validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof StatelessFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    try {
        copyTo(((StatelessFlowFile) flowFile).getDataStream(), out);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public MockFlowFile importFrom(final Path path, final boolean keepSourceFile, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (path == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;
    MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        Files.copy(path, baos);
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }

    newFlowFile.setData(baos.toByteArray());
    newFlowFile = putAttribute(newFlowFile, CoreAttributes.FILENAME.key(), path.getFileName().toString());
    return newFlowFile;
}
 

@Override
public MockFlowFile importFrom(final InputStream in, FlowFile flowFile) {
    flowFile = validateState(flowFile);
    if (in == null || flowFile == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }
    final MockFlowFile mock = (MockFlowFile) flowFile;

    final MockFlowFile newFlowFile = new MockFlowFile(mock.getId(), flowFile);
    currentVersions.put(newFlowFile.getId(), newFlowFile);
    try {
        final byte[] data = readFully(in);
        newFlowFile.setData(data);
        return newFlowFile;
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public void exportTo(FlowFile flowFile, final Path path, final boolean append) {
    flowFile = validateState(flowFile);
    if (flowFile == null || path == null) {
        throw new IllegalArgumentException("argument cannot be null");
    }
    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    final OpenOption mode = append ? StandardOpenOption.APPEND : StandardOpenOption.CREATE;

    try (final OutputStream out = Files.newOutputStream(path, mode)) {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

private void closeStreams(final Map<FlowFile, ? extends Closeable> streamMap, final boolean enforceClosed) {
    final Map<FlowFile, ? extends Closeable> openStreamCopy = new HashMap<>(streamMap); // avoid ConcurrentModificationException by creating a copy of the List
    for (final Map.Entry<FlowFile, ? extends Closeable> entry : openStreamCopy.entrySet()) {
        final FlowFile flowFile = entry.getKey();
        final Closeable openStream = entry.getValue();

        try {
            openStream.close();
        } catch (IOException e) {
            throw new FlowFileAccessException("Failed to close stream for " + flowFile, e);
        }

        if (enforceClosed) {
            throw new FlowFileHandlingException("Cannot commit session because the following streams were created via "
                + "calls to ProcessSession.read(FlowFile) or ProcessSession.write(FlowFile) and never closed: " + streamMap);
        }
    }
}
 

@Override
public void exportTo(FlowFile flowFile, final OutputStream out) {
    flowFile = validateState(flowFile);
    if (flowFile == null || out == null) {
        throw new IllegalArgumentException("arguments cannot be null");
    }

    if (!(flowFile instanceof MockFlowFile)) {
        throw new IllegalArgumentException("Cannot export a flow file that I did not create");
    }

    final MockFlowFile mock = (MockFlowFile) flowFile;

    try {
        out.write(mock.getData());
    } catch (final IOException e) {
        throw new FlowFileAccessException(e.toString(), e);
    }
}
 

@Override
public void reset() throws IOException {
    try {
        super.reset();
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not reset stream from " + flowFile, ioe);
    }
}
 

@Override
public long skip(final long n) throws IOException {
    try {
        final long count = super.skip(n);
        bytesConsumed += count;
        return count;
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not skip data in " + flowFile, ioe);
    }
}
 

@Test
public void testFlowFileAccessExceptionGoesToFailure() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    AmazonS3Exception amazonException = new AmazonS3Exception("testing");
    Mockito.doThrow(new FlowFileAccessException("testing nested", amazonException)).when(mockS3Client).getObject(Mockito.any());

    runner.run(1);

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_FAILURE, 1);
}
 

public StatelessFlowFile(final StatelessFlowFile toCopy, boolean materializeContent) {
    this(materializeContent);
    this.id = toCopy.id;

    attributes.putAll(toCopy.getAttributes());
    this.penalized = toCopy.isPenalized();

    try {
        this.setData(toCopy.getDataArray());
    } catch (IOException e) {
        throw new FlowFileAccessException("Exception creating FlowFile", e);
    }
}
 

@Test
public void testErrorQueue() throws IOException {
    final List<ListenSyslog.RawSyslogEvent> msgs = new ArrayList<>();
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01"));
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01"));

    // Add message that will throw a FlowFileAccessException the first time that we attempt to read
    // the contents but will succeed the second time.
    final AtomicInteger getMessageAttempts = new AtomicInteger(0);
    msgs.add(new ListenSyslog.RawSyslogEvent(VALID_MESSAGE.getBytes(), "sender-01") {
        @Override
        public byte[] getData() {
            final int attempts = getMessageAttempts.incrementAndGet();
            if (attempts == 1) {
                throw new FlowFileAccessException("Unit test failure");
            } else {
                return VALID_MESSAGE.getBytes();
            }
        }
    });

    final CannedMessageProcessor proc = new CannedMessageProcessor(msgs);
    final TestRunner runner = TestRunners.newTestRunner(proc);
    runner.setProperty(ListenSyslog.MAX_BATCH_SIZE, "5");
    runner.setProperty(ListenSyslog.PROTOCOL, ListenSyslog.UDP_VALUE.getValue());
    runner.setProperty(ListenSyslog.PORT, "0");
    runner.setProperty(ListenSyslog.PARSE_MESSAGES, "false");

    runner.run();
    assertEquals(1, proc.getErrorQueueSize());
    runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);
    runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0).assertContentEquals(VALID_MESSAGE + "\n" + VALID_MESSAGE);

    // running again should pull from the error queue
    runner.clearTransferState();
    runner.run();
    runner.assertAllFlowFilesTransferred(ListenSyslog.REL_SUCCESS, 1);
    runner.getFlowFilesForRelationship(ListenSyslog.REL_SUCCESS).get(0).assertContentEquals(VALID_MESSAGE);
}
 

@Override
public void write(final int b) throws IOException {
    try {
        out.write(b);
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not write to " + flowFile, ioe);
    }
}
 

@Override
public void write(final byte[] b) throws IOException {
    try {
        out.write(b);
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not write to " + flowFile, ioe);
    }
}
 

@Override
public void flush() throws IOException {
    try {
        out.flush();
    } catch (final IOException ioe) {
        throw new FlowFileAccessException("Could not flush OutputStream for " + flowFile, ioe);
    }
}
 

@Test
public void testHandlingOfMultipleFlowFilesWithSameId() {
    // Add two FlowFiles with the same ID
    for (int i=0; i < 2; i++) {
        final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder()
            .id(1000L)
            .addAttribute("uuid", "12345678-1234-1234-1234-123456789012")
            .entryDate(System.currentTimeMillis())
            .size(0L)
            .build();

        flowFileQueue.put(flowFileRecord);
    }

    final Relationship relationship = new Relationship.Builder().name("A").build();

    FlowFile ff1 = session.get();
    assertNotNull(ff1);

    session.transfer(ff1, relationship);

    try {
        session.get();
        Assert.fail("Should not have been able to poll second FlowFile with same ID");
    } catch (final FlowFileAccessException e) {
        // Expected
    }
}