Java Code Examples for org.apache.kafka.connect.sink.SinkRecord#kafkaOffset()

public PlainPayload(final SinkRecord record) {
    this.key = record.key() == null ? "" : record.key().toString();
    if (record.keySchema() != null)
        this.keySchemaName = record.keySchema().name();

    this.value = record.value() == null ? "" : record.value().toString();
    if (record.valueSchema() != null)
        this.valueSchemaName = record.valueSchema().name();

    this.topic = record.topic();
    this.partition = record.kafkaPartition();
    this.offset = record.kafkaOffset();

    if (record.timestamp() != null)
        this.timestamp = record.timestamp();
    if (record.timestampType() != null)
        this.timestampTypeName = record.timestampType().name;
}
 

@Override
public void put(Collection<SinkRecord> records) {
    try {
        for (SinkRecord sinkRecord : records) {
            TopicPartition topicPartition = new TopicPartition(sinkRecord.topic(), sinkRecord.kafkaPartition());
            PartitionWriter partition = partitionWriters.get(topicPartition);
            partition.append(Record.fromSinkRecord(sinkRecord));
            if (sinkRecord.kafkaOffset() % 100 == 0) {
                log.debug("Backed up Topic {}, Partition {}, up to offset {}", sinkRecord.topic(), sinkRecord.kafkaPartition(), sinkRecord.kafkaOffset());
            }
            if (config.snapShotMode()) {
                currentOffsets.put(topicPartition, sinkRecord.kafkaOffset());
            }
        }

        // Todo: refactor to own worker. E.g. using the scheduler of MM2
        offsetSink.syncConsumerGroups();
        offsetSink.syncOffsets();

        if (config.snapShotMode()) {
            terminateIfCompleted();
        }
    } catch (IOException | SegmentIndex.IndexException | PartitionIndex.IndexException | SegmentWriter.SegmentException e) {
        throw new RuntimeException(e);
    }
}
 

public static SinkRecord project(SinkRecord record, Schema currentSchema, Compatibility compatibility) {
  switch (compatibility) {
    case BACKWARD:
    case FULL:
    case FORWARD:
      Schema sourceSchema = record.valueSchema();
      Object value = record.value();
      if (sourceSchema == currentSchema || sourceSchema.equals(currentSchema)) {
        return record;
      }
      Object projected = SchemaProjector.project(sourceSchema, value, currentSchema);
      return new SinkRecord(record.topic(), record.kafkaPartition(), record.keySchema(),
                            record.key(), currentSchema, projected, record.kafkaOffset());
    default:
      return record;
  }
}
 

private static String documentIdFromKafkaMetadata(SinkRecord record) {
  Object key = record.key();

  if (key instanceof String
      || key instanceof Number
      || key instanceof Boolean) {
    return key.toString();
  }

  if (key instanceof byte[]) {
    return new String((byte[]) key, UTF_8);
  }

  if (key instanceof ByteBuffer) {
    return toString((ByteBuffer) key);
  }

  return record.topic() + "/" + record.kafkaPartition() + "/" + record.kafkaOffset();
}
 

public Payload(final SinkRecord record) {
  if (record.keySchema() != null) {
    this.keySchemaName = record.keySchema().name();
    if (record.keySchema().version() != null ) {
      this.keySchemaVersion = record.keySchema().version().toString();
    }
  }

  if (record.valueSchema() != null) {
    this.valueSchemaName = record.valueSchema().name();
    if (record.valueSchema().version() != null ) {
      this.valueSchemaVersion = record.valueSchema().version().toString();
    }
  }

  this.topic = record.topic();
  this.partition = record.kafkaPartition();
  this.offset = record.kafkaOffset();

  if (record.timestamp() != null) {
    this.timestamp = record.timestamp();
  }
  if (record.timestampType() != null) {
    this.timestampTypeName = record.timestampType().name;
  }

}
 

private void insert(SinkRecord record)
{
  String data = recordService.processRecord(record);
  if (bufferSize == 0)
  {
    firstOffset = record.kafkaOffset();
  }

  stringBuilder.append(data);
  numOfRecord++;
  bufferSize += data.length() * 2; //1 char = 2 bytes
  lastOffset = record.kafkaOffset();
}
 

public static Record fromSinkRecord(SinkRecord sinkRecord) {
    byte[] key = connectDataToBytes(sinkRecord.keySchema(), sinkRecord.key());
    byte[] value = connectDataToBytes(sinkRecord.valueSchema(), sinkRecord.value());
    RecordHeaders recordHeaders = new RecordHeaders();
    for (org.apache.kafka.connect.header.Header connectHeader : sinkRecord.headers()) {
        byte[] headerValue = connectDataToBytes(connectHeader.schema(), connectHeader.value());
        recordHeaders.add(connectHeader.key(), headerValue);
    }
    return new Record(sinkRecord.topic(), sinkRecord.kafkaPartition(), key, value, sinkRecord.kafkaOffset(), sinkRecord.timestamp(), sinkRecord.timestampType(), recordHeaders);
}
 

@Override
public BsonValue generateId(SinkDocument doc, SinkRecord orig) {

   return new BsonString(orig.topic()
                    + DELIMITER + orig.kafkaPartition()
                    + DELIMITER + orig.kafkaOffset());

}
 

private void writeRecord(SinkRecord record) throws IOException {
  long expectedOffset = offset + recordCounter;
  if (offset == -1) {
    offset = record.kafkaOffset();
  } else if (record.kafkaOffset() != expectedOffset) {
    // Currently it's possible to see stale data with the wrong offset after a rebalance when you
    // rewind, which we do since we manage our own offsets. See KAFKA-2894.
    if (!sawInvalidOffset) {
      log.info(
          "Ignoring stale out-of-order record in {}-{}. Has offset {} instead of expected offset {}",
          record.topic(), record.kafkaPartition(), record.kafkaOffset(), expectedOffset);
    }
    sawInvalidOffset = true;
    return;
  }

  if (sawInvalidOffset) {
    log.info(
        "Recovered from stale out-of-order records in {}-{} with offset {}",
        record.topic(), record.kafkaPartition(), expectedOffset);
    sawInvalidOffset = false;
  }

  String encodedPartition = partitioner.encodePartition(record);
  RecordWriter<SinkRecord> writer = getWriter(record, encodedPartition);
  writer.write(record);

  if (!startOffsets.containsKey(encodedPartition)) {
    startOffsets.put(encodedPartition, record.kafkaOffset());
    offsets.put(encodedPartition, record.kafkaOffset());
  } else {
    offsets.put(encodedPartition, record.kafkaOffset());
  }
  recordCounter++;
}
 

private String createFilename(final SinkRecord record) {
    return PREFIX + record.topic() + "-" + record.kafkaPartition() + "-" + record.kafkaOffset();
}
 

@Override
public BsonValue generateId(final SinkDocument doc, final SinkRecord orig) {
  return new BsonString(
      orig.topic() + DELIMITER + orig.kafkaPartition() + DELIMITER + orig.kafkaOffset());
}
 

private String generateId(final SinkRecord record) {
  return record.topic() + ":" + record.kafkaPartition() + ":" + record.kafkaOffset();
}
 

KafkaDocument(SinkRecord record) {
    this.record = record;
    partition = new TopicPartition(record.topic(), record.kafkaPartition());
    offset = record.kafkaOffset();
}