Java Code Examples for org.apache.kafka.connect.sink.SinkRecord#kafkaPartition()

public PlainPayload(final SinkRecord record) {
    this.key = record.key() == null ? "" : record.key().toString();
    if (record.keySchema() != null)
        this.keySchemaName = record.keySchema().name();

    this.value = record.value() == null ? "" : record.value().toString();
    if (record.valueSchema() != null)
        this.valueSchemaName = record.valueSchema().name();

    this.topic = record.topic();
    this.partition = record.kafkaPartition();
    this.offset = record.kafkaOffset();

    if (record.timestamp() != null)
        this.timestamp = record.timestamp();
    if (record.timestampType() != null)
        this.timestampTypeName = record.timestampType().name;
}
 

@Override
public void put(final SinkRecord record) {
    Objects.requireNonNull(record, "record cannot be null");

    final TopicPartition tp = new TopicPartition(record.topic(), record.kafkaPartition());
    final SinkRecord currentHeadRecord = currentHeadRecords.computeIfAbsent(tp, ignored -> record);
    final String recordKey = generateRecordKey(tp, currentHeadRecord);

    if (shouldCreateNewFile(recordKey)) {
        // Create new file using this record as the head record.
        currentHeadRecords.put(tp, record);
        final String newRecordKey = generateRecordKey(tp, record);
        fileBuffers.computeIfAbsent(newRecordKey, ignored -> new ArrayList<>()).add(record);
    } else {
        fileBuffers.computeIfAbsent(recordKey, ignored -> new ArrayList<>()).add(record);
    }
}
 

@Override
public void put(Collection<SinkRecord> records) {
    try {
        for (SinkRecord sinkRecord : records) {
            TopicPartition topicPartition = new TopicPartition(sinkRecord.topic(), sinkRecord.kafkaPartition());
            PartitionWriter partition = partitionWriters.get(topicPartition);
            partition.append(Record.fromSinkRecord(sinkRecord));
            if (sinkRecord.kafkaOffset() % 100 == 0) {
                log.debug("Backed up Topic {}, Partition {}, up to offset {}", sinkRecord.topic(), sinkRecord.kafkaPartition(), sinkRecord.kafkaOffset());
            }
            if (config.snapShotMode()) {
                currentOffsets.put(topicPartition, sinkRecord.kafkaOffset());
            }
        }

        // Todo: refactor to own worker. E.g. using the scheduler of MM2
        offsetSink.syncConsumerGroups();
        offsetSink.syncOffsets();

        if (config.snapShotMode()) {
            terminateIfCompleted();
        }
    } catch (IOException | SegmentIndex.IndexException | PartitionIndex.IndexException | SegmentWriter.SegmentException e) {
        throw new RuntimeException(e);
    }
}
 

public static SinkRecord project(SinkRecord record, Schema currentSchema, Compatibility compatibility) {
  switch (compatibility) {
    case BACKWARD:
    case FULL:
    case FORWARD:
      Schema sourceSchema = record.valueSchema();
      Object value = record.value();
      if (sourceSchema == currentSchema || sourceSchema.equals(currentSchema)) {
        return record;
      }
      Object projected = SchemaProjector.project(sourceSchema, value, currentSchema);
      return new SinkRecord(record.topic(), record.kafkaPartition(), record.keySchema(),
                            record.key(), currentSchema, projected, record.kafkaOffset());
    default:
      return record;
  }
}
 

@Override
public void put(Collection<SinkRecord> sinkRecords) {
    if (writer == null) {
        return;
    }

    putTimer.start();

    for (SinkRecord record : sinkRecords) {
        TopicPartition partition = new TopicPartition(record.topic(),
                record.kafkaPartition());
        latestOffsetPut.put(partition, record.kafkaOffset());
        buffer.add(record);
        monitor.increment();

        if (log.isDebugEnabled()) {
            log.debug("{} --> {}", partition, record.kafkaOffset());
        }
    }

    putTimer.stop();
}
 

private static String documentIdFromKafkaMetadata(SinkRecord record) {
  Object key = record.key();

  if (key instanceof String
      || key instanceof Number
      || key instanceof Boolean) {
    return key.toString();
  }

  if (key instanceof byte[]) {
    return new String((byte[]) key, UTF_8);
  }

  if (key instanceof ByteBuffer) {
    return toString((ByteBuffer) key);
  }

  return record.topic() + "/" + record.kafkaPartition() + "/" + record.kafkaOffset();
}
 

public Payload(final SinkRecord record) {
  if (record.keySchema() != null) {
    this.keySchemaName = record.keySchema().name();
    if (record.keySchema().version() != null ) {
      this.keySchemaVersion = record.keySchema().version().toString();
    }
  }

  if (record.valueSchema() != null) {
    this.valueSchemaName = record.valueSchema().name();
    if (record.valueSchema().version() != null ) {
      this.valueSchemaVersion = record.valueSchema().version().toString();
    }
  }

  this.topic = record.topic();
  this.partition = record.kafkaPartition();
  this.offset = record.kafkaOffset();

  if (record.timestamp() != null) {
    this.timestamp = record.timestamp();
  }
  if (record.timestampType() != null) {
    this.timestampTypeName = record.timestampType().name;
  }

}
 

public static Record fromSinkRecord(SinkRecord sinkRecord) {
    byte[] key = connectDataToBytes(sinkRecord.keySchema(), sinkRecord.key());
    byte[] value = connectDataToBytes(sinkRecord.valueSchema(), sinkRecord.value());
    RecordHeaders recordHeaders = new RecordHeaders();
    for (org.apache.kafka.connect.header.Header connectHeader : sinkRecord.headers()) {
        byte[] headerValue = connectDataToBytes(connectHeader.schema(), connectHeader.value());
        recordHeaders.add(connectHeader.key(), headerValue);
    }
    return new Record(sinkRecord.topic(), sinkRecord.kafkaPartition(), key, value, sinkRecord.kafkaOffset(), sinkRecord.timestamp(), sinkRecord.timestampType(), recordHeaders);
}
 

@Override
public BsonValue generateId(SinkDocument doc, SinkRecord orig) {

   return new BsonString(orig.topic()
                    + DELIMITER + orig.kafkaPartition()
                    + DELIMITER + orig.kafkaOffset());

}
 

private String createFilename(final SinkRecord record) {
    return PREFIX + record.topic() + "-" + record.kafkaPartition() + "-" + record.kafkaOffset();
}
 

@Override
public BsonValue generateId(final SinkDocument doc, final SinkRecord orig) {
  return new BsonString(
      orig.topic() + DELIMITER + orig.kafkaPartition() + DELIMITER + orig.kafkaOffset());
}
 

private String generateId(final SinkRecord record) {
  return record.topic() + ":" + record.kafkaPartition() + ":" + record.kafkaOffset();
}
 

KafkaDocument(SinkRecord record) {
    this.record = record;
    partition = new TopicPartition(record.topic(), record.kafkaPartition());
    offset = record.kafkaOffset();
}