Java Code Examples for org.elasticsearch.action.index.IndexRequestBuilder#setTTL()

@Override
public void addEvent(Event event, IndexNameBuilder indexNameBuilder,
                     String indexType, long ttlMs) throws Exception {
    if (bulkRequestBuilder == null) {
        bulkRequestBuilder = client.prepareBulk();
    }

    IndexRequestBuilder indexRequestBuilder;
    if (indexRequestBuilderFactory == null) {
        indexRequestBuilder = client
                .prepareIndex(indexNameBuilder.getIndexName(event), indexType)
                .setSource(serializer.getContentBuilder(event).bytes());
    } else {
        indexRequestBuilder = indexRequestBuilderFactory.createIndexRequest(
                client, indexNameBuilder.getIndexPrefix(event), indexType, event);
    }

    if (ttlMs > 0) {
        indexRequestBuilder.setTTL(ttlMs);
    }
    bulkRequestBuilder.add(indexRequestBuilder);
}
 

public Optional<BulkResult> indexData(ElasticDataPointer targetDataPointer, SearchHit[] hits) {
  BulkRequestBuilder bulkRequest = createBulkRequestBuilder();

  for (SearchHit hit : hits) {
    Map<String, Object> source = hit.getSource();

    IndexRequestBuilder requestBuilder = prepareIndex(targetDataPointer.getIndexName(), targetDataPointer
        .getTypeName(), hit.getId(), source, hit.getIndex());
    if (hit.getFields().get("_ttl") != null) {
      requestBuilder.setTTL(hit.getFields().get("_ttl").value());
    }
    if (hit.getFields().get("_routing") != null) {
      requestBuilder.setRouting(hit.getFields().get("_routing").value());
    }
    requestBuilder.setSource(source);
    bulkRequest.add(requestBuilder);
  }
  return executeBulk(hits.length, bulkRequest);
}
 

@Override
public void addEvent(Event event, IndexNameBuilder indexNameBuilder,
    String indexType, long ttlMs) throws Exception {
  if (bulkRequestBuilder == null) {
    bulkRequestBuilder = client.prepareBulk();
  }

  IndexRequestBuilder indexRequestBuilder = null;
  if (indexRequestBuilderFactory == null) {
    indexRequestBuilder = client
        .prepareIndex(indexNameBuilder.getIndexName(event), indexType)
        .setSource(serializer.getContentBuilder(event).bytes());
  } else {
    indexRequestBuilder = indexRequestBuilderFactory.createIndexRequest(
        client, indexNameBuilder.getIndexPrefix(event), indexType, event);
  }

  if (ttlMs > 0) {
    indexRequestBuilder.setTTL(ttlMs);
  }
  bulkRequestBuilder.add(indexRequestBuilder);
}
 

@Override
public Status process() throws EventDeliveryException {
  logger.debug("processing...");
  Status status = Status.READY;
  Channel channel = getChannel();
  Transaction txn = channel.getTransaction();
  try {
    txn.begin();
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (int i = 0; i < batchSize; i++) {
      Event event = channel.take();

      if (event == null) {
        break;
      }

      IndexRequestBuilder indexRequest =
          indexRequestFactory.createIndexRequest(
              client, indexName, indexType, event);

      if (ttlMs > 0) {
        indexRequest.setTTL(ttlMs);
      }

      bulkRequest.add(indexRequest);
    }

    int size = bulkRequest.numberOfActions();
    if (size <= 0) {
      sinkCounter.incrementBatchEmptyCount();
      counterGroup.incrementAndGet("channel.underflow");
      status = Status.BACKOFF;
    } else {
      if (size < batchSize) {
        sinkCounter.incrementBatchUnderflowCount();
        status = Status.BACKOFF;
      } else {
        sinkCounter.incrementBatchCompleteCount();
      }

      sinkCounter.addToEventDrainAttemptCount(size);

      BulkResponse bulkResponse = bulkRequest.execute().actionGet();
      if (bulkResponse.hasFailures()) {
        throw new EventDeliveryException(bulkResponse.buildFailureMessage());
      }
    }
    txn.commit();
    sinkCounter.addToEventDrainSuccessCount(size);
    counterGroup.incrementAndGet("transaction.success");
  } catch (Throwable ex) {
    try {
      txn.rollback();
      counterGroup.incrementAndGet("transaction.rollback");
    } catch (Exception ex2) {
      logger.error(
          "Exception in rollback. Rollback might not have been successful.",
          ex2);
    }

    if (ex instanceof Error || ex instanceof RuntimeException) {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      Throwables.propagate(ex);
    } else {
      logger.error("Failed to commit transaction. Transaction rolled back.",
          ex);
      throw new EventDeliveryException(
          "Failed to commit transaction. Transaction rolled back.", ex);
    }
  } finally {
    txn.close();
  }
  return status;
}
 

/**
 * Emits records to elasticsearch.
 * 1. Adds each record to a bulk index request, conditionally adding version, ttl or create if they were set in the
 * transformer.
 * 2. Executes the bulk request, returning any record specific failures to be retried by the connector library
 * pipeline, unless
 * outlined below.
 * 
 * Record specific failures (noted in the failure.getMessage() string)
 * - DocumentAlreadyExistsException means the record has create set to true, but a document already existed at the
 * specific index/type/id.
 * - VersionConflictEngineException means the record has a specific version number that did not match what existed
 * in elasticsearch.
 * To guarantee in order processing by the connector, when putting data use the same partition key for objects going
 * to the same
 * index/type/id and set sequence number for ordering.
 * - In either case, the emitter will assume that the record would fail again in the future and thus will not return
 * the record to
 * be retried.
 * 
 * Bulk request failures
 * - NoNodeAvailableException means the TransportClient could not connect to the cluster.
 * - A general Exception catches any other unexpected behavior.
 * - In either case the emitter will continue making attempts until the issue has been resolved. This is to ensure
 * that no data
 * loss occurs and simplifies restarting the application once issues have been fixed.
 */
@Override
public List<ElasticsearchObject> emit(UnmodifiableBuffer<ElasticsearchObject> buffer) throws IOException {
    List<ElasticsearchObject> records = buffer.getRecords();
    if (records.isEmpty()) {
        return Collections.emptyList();
    }

    BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
    for (ElasticsearchObject record : records) {
        IndexRequestBuilder indexRequestBuilder =
                elasticsearchClient.prepareIndex(record.getIndex(), record.getType(), record.getId());
        indexRequestBuilder.setSource(record.getSource());
        Long version = record.getVersion();
        if (version != null) {
            indexRequestBuilder.setVersion(version);
        }
        Long ttl = record.getTtl();
        if (ttl != null) {
            indexRequestBuilder.setTTL(ttl);
        }
        Boolean create = record.getCreate();
        if (create != null) {
            indexRequestBuilder.setCreate(create);
        }
        bulkRequest.add(indexRequestBuilder);
    }

    while (true) {
        try {
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();

            BulkItemResponse[] responses = bulkResponse.getItems();
            List<ElasticsearchObject> failures = new ArrayList<ElasticsearchObject>();
            int numberOfSkippedRecords = 0;
            for (int i = 0; i < responses.length; i++) {
                if (responses[i].isFailed()) {
                    LOG.error("Record failed with message: " + responses[i].getFailureMessage());
                    Failure failure = responses[i].getFailure();
                    if (failure.getMessage().contains("DocumentAlreadyExistsException")
                            || failure.getMessage().contains("VersionConflictEngineException")) {
                        numberOfSkippedRecords++;
                    } else {
                        failures.add(records.get(i));
                    }
                }
            }
            LOG.info("Emitted " + (records.size() - failures.size() - numberOfSkippedRecords)
                    + " records to Elasticsearch");
            if (!failures.isEmpty()) {
                printClusterStatus();
                LOG.warn("Returning " + failures.size() + " records as failed");
            }
            return failures;
        } catch (NoNodeAvailableException nnae) {
            LOG.error("No nodes found at " + elasticsearchEndpoint + ":" + elasticsearchPort + ". Retrying in "
                    + BACKOFF_PERIOD + " milliseconds", nnae);
            sleep(BACKOFF_PERIOD);
        } catch (Exception e) {
            LOG.error("ElasticsearchEmitter threw an unexpected exception ", e);
            sleep(BACKOFF_PERIOD);
        }
    }

}