org.elasticsearch.action.bulk.BulkItemResponse.Failure Java Examples

private void flushBulk() {
	BulkResponse bulkResponse = bulkRequest.execute().actionGet(); // flush
	if (bulkResponse.hasFailures()) { // log failures
		for (BulkItemResponse r : bulkResponse.getItems()) {
			LOG.error(r.getFailureMessage());
			Failure failure = r.getFailure();
			// e.g. when ES server is overloaded
			throw new ElasticsearchException(failure.toString());
		}
	}
}
 

public void addAll(final String index, final List<Map<String, Object>> docList,
        final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (final Map<String, Object> doc : docList) {
        final Object id = doc.remove(fessConfig.getIndexFieldId());
        final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
        options.accept(doc, builder);
        bulkRequestBuilder.add(builder);
    }
    final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
    if (response.hasFailures()) {
        if (logger.isDebugEnabled()) {
            final List<DocWriteRequest<?>> requests = bulkRequestBuilder.request().requests();
            final BulkItemResponse[] items = response.getItems();
            if (requests.size() == items.length) {
                for (int i = 0; i < requests.size(); i++) {
                    final BulkItemResponse resp = items[i];
                    if (resp.isFailed() && resp.getFailure() != null) {
                        final DocWriteRequest<?> req = requests.get(i);
                        final Failure failure = resp.getFailure();
                        logger.debug("Failed Request: {}\n=>{}", req, failure.getMessage());
                    }
                }
            }
        }
        throw new FessEsClientException(response.buildFailureMessage());
    }
}
 

/**
 * Emits records to elasticsearch.
 * 1. Adds each record to a bulk index request, conditionally adding version, ttl or create if they were set in the
 * transformer.
 * 2. Executes the bulk request, returning any record specific failures to be retried by the connector library
 * pipeline, unless
 * outlined below.
 * 
 * Record specific failures (noted in the failure.getMessage() string)
 * - DocumentAlreadyExistsException means the record has create set to true, but a document already existed at the
 * specific index/type/id.
 * - VersionConflictEngineException means the record has a specific version number that did not match what existed
 * in elasticsearch.
 * To guarantee in order processing by the connector, when putting data use the same partition key for objects going
 * to the same
 * index/type/id and set sequence number for ordering.
 * - In either case, the emitter will assume that the record would fail again in the future and thus will not return
 * the record to
 * be retried.
 * 
 * Bulk request failures
 * - NoNodeAvailableException means the TransportClient could not connect to the cluster.
 * - A general Exception catches any other unexpected behavior.
 * - In either case the emitter will continue making attempts until the issue has been resolved. This is to ensure
 * that no data
 * loss occurs and simplifies restarting the application once issues have been fixed.
 */
@Override
public List<ElasticsearchObject> emit(UnmodifiableBuffer<ElasticsearchObject> buffer) throws IOException {
    List<ElasticsearchObject> records = buffer.getRecords();
    if (records.isEmpty()) {
        return Collections.emptyList();
    }

    BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
    for (ElasticsearchObject record : records) {
        IndexRequestBuilder indexRequestBuilder =
                elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
        indexRequestBuilder.setSource(record.getSource());
        Long version = record.getVersion();
        if (version != null) {
            indexRequestBuilder.setVersion(version);
        }
        Long ttl = record.getTtl();
        if (ttl != null) {
            indexRequestBuilder.setTTL(ttl);
        }
        Boolean create = record.getCreate();
        if (create != null) {
            indexRequestBuilder.setCreate(create);
        }
        bulkRequest.add(indexRequestBuilder);
    }

    while (true) {
        try {
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();

            BulkItemResponse[] responses = bulkResponse.getItems();
            List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
            int numberOfSkippedRecords = 0;
            for (int i = 0; i < responses.length; i++) {
                if (responses[i].isFailed()) {
                    LOG.error("Record failed with message: " + responses[i].getFailureMessage());
                    Failure failure = responses[i].getFailure();
                    if (failure.getMessage().contains("DocumentAlreadyExistsException")
                            || failure.getMessage().contains("VersionConflictEngineException")) {
                        numberOfSkippedRecords++;
                    } else {
                        failures.add(records.get(i));
                    }
                }
            }
            LOG.info("Emitted " + (records.size() - failures.size() - numberOfSkippedRecords)
                    + " records to Elasticsearch");
            if (!failures.isEmpty()) {
                printClusterStatus();
                LOG.warn("Returning " + failures.size() + " records as failed");
            }
            return failures;
        } catch (NoNodeAvailableException nnae) {
            LOG.error("No nodes found at " + elasticsearchEndpoint + ":" + elasticsearchPort + ". Retrying in "
                    + BACKOFF_PERIOD + " milliseconds", nnae);
            sleep(BACKOFF_PERIOD);
        } catch (Exception e) {
            LOG.error("ElasticsearchEmitter threw an unexpected exception ", e);
            sleep(BACKOFF_PERIOD);
        }
    }

}
 

/**
 * Set the 2nd record in the passed in list to fail.
 * 
 * Assert that only 1 record is returned, and that it is equal to the 2nd object in the list.
 * 
 * @throws IOException
 */
@Test
public void testRecordFails() throws IOException {

    List<ElasticsearchObject> records = new ArrayList<ElasticsearchObject>();
    ElasticsearchObject r1 = createMockRecordAndSetExpectations("sample-index", "type", "1", "{\"name\":\"Mike\"}");
    records.add(r1);
    // this will be the failing record.
    ElasticsearchObject r2 = createMockRecordAndSetExpectations("sample-index", "type", "2",
            "{\"name\":\"Mike\",\"badJson\":\"forgotendingquote}");
    records.add(r2);
    ElasticsearchObject r3 = createMockRecordAndSetExpectations("sample-index", "type", "3", "{\"name\":\"Mike\"}");
    records.add(r3);

    // mock building and executing the request
    mockBuildingAndExecutingRequest(records);

    // Verify that there is a response for each record, and that each gets touched.
    BulkItemResponse[] responses = new BulkItemResponse[records.size()];
    for (int i = 0; i < responses.length; i++) {
        responses[i] = createMock(BulkItemResponse.class);
        // define behavior for a failing record
        if (i == 1) {
            expect(responses[i].isFailed()).andReturn(true);
            expect(responses[i].getFailureMessage()).andReturn("bad json error message");
            Failure failure = new Failure("index", "type", "id", "foo failure message", RestStatus.BAD_REQUEST);
            expect(responses[i].getFailure()).andReturn(failure);
        } else {
            expect(responses[i].isFailed()).andReturn(false);
        }
        replay(responses[i]);
    }

    // verify admin client gets used to check cluster status. this case, yellow
    expect(mockBulkResponse.getItems()).andReturn(responses);
    AdminClient mockAdminClient = createMock(AdminClient.class);
    expect(elasticsearchClientMock.admin()).andReturn(mockAdminClient);
    ClusterAdminClient mockClusterAdminClient = createMock(ClusterAdminClient.class);
    expect(mockAdminClient.cluster()).andReturn(mockClusterAdminClient);
    ClusterHealthRequestBuilder mockHealthRequestBuilder = createMock(ClusterHealthRequestBuilder.class);
    expect(mockClusterAdminClient.prepareHealth()).andReturn(mockHealthRequestBuilder);
    ListenableActionFuture mockHealthFuture = createMock(ListenableActionFuture.class);
    expect(mockHealthRequestBuilder.execute()).andReturn(mockHealthFuture);
    ClusterHealthResponse mockResponse = createMock(ClusterHealthResponse.class);
    expect(mockHealthFuture.actionGet()).andReturn(mockResponse);
    expect(mockResponse.getStatus()).andReturn(ClusterHealthStatus.YELLOW);
    expectLastCall().times(2);

    replay(elasticsearchClientMock, r1, r2, r3, buffer, mockBulkBuilder, mockIndexBuilder, mockFuture,
            mockBulkResponse, mockAdminClient, mockClusterAdminClient, mockHealthRequestBuilder, mockHealthFuture,
            mockResponse);

    List<ElasticsearchObject> failures = emitter.emit(buffer);
    assertTrue(failures.size() == 1);
    // the emitter should return the exact object that failed
    assertEquals(failures.get(0), records.get(1));

    verify(elasticsearchClientMock, r1, r2, r3, buffer, mockBulkBuilder, mockIndexBuilder, mockFuture,
            mockBulkResponse, mockAdminClient, mockClusterAdminClient, mockHealthRequestBuilder, mockHealthFuture,
            mockResponse);
    verifyRecords(records);
    verifyResponses(responses);
}