Java Code Examples for org.apache.kafka.connect.data.Struct#put()

public static void addRecord(Collection<SinkRecord> records, Map<String, ?> values) {
  Struct valueStruct = new Struct(EventConverter.VALUE_SCHEMA);
  for (Map.Entry<String, ?> entry : values.entrySet()) {
    valueStruct.put(entry.getKey(), entry.getValue());
  }

  records.add(
      new SinkRecord(
          "topic",
          1,
          EventConverter.KEY_SCHEMA,
          null,
          EventConverter.VALUE_SCHEMA,
          valueStruct,
          1L
      )
  );
}
 

static <T> void setFieldValue(JsonNode messageNode, Struct struct, String fieldName, Class<T> cls) {
  T structValue = null;

  if (messageNode.has(fieldName)) {
    JsonNode valueNode = messageNode.get(fieldName);

    if (String.class.equals(cls) && valueNode.isObject()) {
      try {
        structValue = (T) ObjectMapperFactory.INSTANCE.writeValueAsString(valueNode);
      } catch (JsonProcessingException e) {
        throw new IllegalStateException(e);
      }
    } else if (!valueNode.isNull()) {
      structValue = ObjectMapperFactory.INSTANCE.convertValue(valueNode, cls);
    }
  }

  struct.put(fieldName, structValue);
}
 

@Override
public void convert(LogEntry logEntry, Struct struct) {
  final LocalDate date = (LocalDate) logEntry.fieldData().get(this.dateField);
  final LocalTime time = (LocalTime) logEntry.fieldData().get(this.timeField);

  final Object value;

  if (null == date || null == time) {
    value = null;
  } else {
    final Instant instant = time.atDate(date).toInstant(ZoneOffset.UTC);
    value = Date.from(instant);
  }
  struct.put(this.field, value);
}
 

@Override
protected List<SourceRecord> process() {
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);

  while (this.iterator.hasNext() && records.size() < this.config.batchSize) {
    JsonNode node = next();

    Struct valueStruct = new Struct(this.config.valueSchema);
    Struct keyStruct = new Struct(this.config.keySchema);
    log.trace("process() - input = {}", node);
    for (Field field : this.config.valueSchema.fields()) {
      JsonNode fieldNode = node.get(field.name());
      log.trace("process() - field: {} input = '{}'", field.name(), fieldNode);
      Object fieldValue;
      try {
        fieldValue = this.parser.parseJsonNode(field.schema(), fieldNode);
        log.trace("process() - field: {} output = '{}'", field.name(), fieldValue);
        valueStruct.put(field, fieldValue);

        Field keyField = this.config.keySchema.field(field.name());
        if (null != keyField) {
          log.trace("process() - Setting key field '{}' to '{}'", keyField.name(), fieldValue);
          keyStruct.put(keyField, fieldValue);
        }
      } catch (Exception ex) {
        String message = String.format("Exception thrown while parsing data for '%s'. linenumber=%s", field.name(), this.recordOffset());
        throw new DataException(message, ex);
      }
    }

    addRecord(
        records,
        new SchemaAndValue(keyStruct.schema(), keyStruct),
        new SchemaAndValue(valueStruct.schema(), valueStruct)
    );
  }

  return records;
}
 

public Struct build() {
  log.trace("build() - Creating struct for {}", this.schema);
  Struct struct = new Struct(this.schema);
  for (KeyValue keyValue : this.fieldValues) {
    log.trace("build() - Setting field value for '{}'", keyValue.name);
    struct.put(keyValue.name, keyValue.value());
  }
  struct.validate();
  return struct;
}
 

public void convert(LogEntry logEntry, Struct struct) {
  final Object input = logEntry.fieldData().get(this.logFieldName);
  final Object output;
  if (null == input) {
    output = null;
  } else {
    output = convert(input);
  }

  log.trace("convert() - Setting {} to {}", field.name(), output);
  struct.put(this.field, output);
}
 

@Override
public Struct record(Schema schema) {
    Struct struct = new Struct(schema);
    for (Field field : schema.fields()) {
        Schema cellSchema = getFieldSchema(field.name(), schema);
        CellData cellData = cellMap.get(field.name());
        // only add the cell if it is not null
        if (cellData != null) {
            struct.put(field.name(), cellData.record(cellSchema));
        }
    }
    return struct;
}
 

public Struct buildKey() {
    if (keySchema == null) {
        return null;
    }

    List<CellData> primary = rowData.getPrimary();
    Struct struct = new Struct(keySchema);
    for (CellData cellData : primary) {
        struct.put(cellData.name, cellData.value);
    }
    return struct;
}
 

@Override
public Struct struct(SourceInfo sourceInfo) {
    final Struct ret = super.commonStruct(sourceInfo)
            .put(SourceInfo.SCHEMA_NAME_KEY, sourceInfo.getTableId().schema())
            .put(SourceInfo.TABLE_NAME_KEY, sourceInfo.getTableId().table());

    if (sourceInfo.getChangeLsn() != null && sourceInfo.getChangeLsn().isAvailable()) {
        ret.put(SourceInfo.CHANGE_LSN_KEY, sourceInfo.getChangeLsn().toString());
    }
    if (sourceInfo.getCommitLsn() != null && sourceInfo.getCommitLsn().isAvailable()) {
        ret.put(SourceInfo.COMMIT_LSN_KEY, sourceInfo.getCommitLsn().toString());
    }
    return ret;
}
 

public Struct buildRecordValue(Issue issue){

        // Issue top level fields
        Struct valueStruct = new Struct(VALUE_SCHEMA)
                .put(URL_FIELD, issue.getUrl())
                .put(TITLE_FIELD, issue.getTitle())
                .put(CREATED_AT_FIELD, Date.from(issue.getCreatedAt()))
                .put(UPDATED_AT_FIELD, Date.from(issue.getUpdatedAt()))
                .put(NUMBER_FIELD, issue.getNumber())
                .put(STATE_FIELD, issue.getState());

        // User is mandatory
        User user = issue.getUser();
        Struct userStruct = new Struct(USER_SCHEMA)
                .put(USER_URL_FIELD, user.getUrl())
                .put(USER_ID_FIELD, user.getId())
                .put(USER_LOGIN_FIELD, user.getLogin());
        valueStruct.put(USER_FIELD, userStruct);

        // Pull request is optional
        PullRequest pullRequest = issue.getPullRequest();
        if (pullRequest != null) {
            Struct prStruct = new Struct(PR_SCHEMA)
                    .put(PR_URL_FIELD, pullRequest.getUrl())
                    .put(PR_HTML_URL_FIELD, pullRequest.getHtmlUrl());
            valueStruct.put(PR_FIELD, prStruct);
        }

        return valueStruct;
    }
 

/**
 * @return true to publish the message, or false to skip it
 */
protected boolean buildValue(SourceHandlerParams params, SourceRecordBuilder builder) {
  final DocumentEvent docEvent = params.documentEvent();
  final DocumentEvent.Type type = docEvent.type();

  final Struct record = new Struct(Schemas.VALUE_DEFAULT_SCHEMA);
  record.put("event", type.schemaName());

  record.put("bucket", docEvent.bucket());
  record.put("partition", docEvent.partition());
  record.put("vBucketUuid", docEvent.partitionUuid());
  record.put("key", docEvent.key());
  record.put("cas", docEvent.cas());
  record.put("bySeqno", docEvent.bySeqno());
  record.put("revSeqno", docEvent.revisionSeqno());

  final MutationMetadata mutation = docEvent.mutationMetadata().orElse(null);
  if (mutation != null) {
    record.put("expiration", mutation.expiry());
    record.put("flags", mutation.flags());
    record.put("lockTime", mutation.lockTime());
    record.put("content", docEvent.content());

  } else if (type != DocumentEvent.Type.DELETION && type != DocumentEvent.Type.EXPIRATION) {
    LOGGER.warn("unexpected event type: {}", type);
    return false;
  }

  builder.value(Schemas.VALUE_DEFAULT_SCHEMA, record);
  return true;
}
 

public Struct build() {
  final Schema logRecordSchema = LogRecordBuilder.baseSchemaBuilder()
      .field(FIELD_MESSAGE, messageWithSchema.schema())
      .build();
  final Struct logRecord = new Struct(logRecordSchema);
  logRecord.put(FIELD_LOGGER, loggerName);
  logRecord.put(FIELD_LEVEL, level);
  logRecord.put(FIELD_TIME, timeMs);
  logRecord.put(FIELD_MESSAGE, messageWithSchema.value());
  return logRecord;
}
 

public static void convert(StatusDeletionNotice statusDeletionNotice, Struct struct) {
  struct.put("StatusId", statusDeletionNotice.getStatusId());
  struct.put("UserId", statusDeletionNotice.getUserId());
}
 

public static void convertKey(Status status, Struct struct) {
  struct.put("Id", status.getId());
}
 

public static void convert(User user, Struct struct) {
  struct
      .put("Id", user.getId())
      .put("Name", user.getName())
      .put("ScreenName", user.getScreenName())
      .put("Location", user.getLocation())
      .put("Description", user.getDescription())
      .put("ContributorsEnabled", user.isContributorsEnabled())
      .put("ProfileImageURL", user.getProfileImageURL())
      .put("BiggerProfileImageURL", user.getBiggerProfileImageURL())
      .put("MiniProfileImageURL", user.getMiniProfileImageURL())
      .put("OriginalProfileImageURL", user.getOriginalProfileImageURL())
      .put("ProfileImageURLHttps", user.getProfileImageURLHttps())
      .put("BiggerProfileImageURLHttps", user.getBiggerProfileImageURLHttps())
      .put("MiniProfileImageURLHttps", user.getMiniProfileImageURLHttps())
      .put("OriginalProfileImageURLHttps", user.getOriginalProfileImageURLHttps())
      .put("DefaultProfileImage", user.isDefaultProfileImage())
      .put("URL", user.getURL())
      .put("Protected", user.isProtected())
      .put("FollowersCount", user.getFollowersCount())
      .put("ProfileBackgroundColor", user.getProfileBackgroundColor())
      .put("ProfileTextColor", user.getProfileTextColor())
      .put("ProfileLinkColor", user.getProfileLinkColor())
      .put("ProfileSidebarFillColor", user.getProfileSidebarFillColor())
      .put("ProfileSidebarBorderColor", user.getProfileSidebarBorderColor())
      .put("ProfileUseBackgroundImage", user.isProfileUseBackgroundImage())
      .put("DefaultProfile", user.isDefaultProfile())
      .put("ShowAllInlineMedia", user.isShowAllInlineMedia())
      .put("FriendsCount", user.getFriendsCount())
      .put("CreatedAt", user.getCreatedAt())
      .put("FavouritesCount", user.getFavouritesCount())
      .put("UtcOffset", user.getUtcOffset())
      .put("TimeZone", user.getTimeZone())
      .put("ProfileBackgroundImageURL", user.getProfileBackgroundImageURL())
      .put("ProfileBackgroundImageUrlHttps", user.getProfileBackgroundImageUrlHttps())
      .put("ProfileBannerURL", user.getProfileBannerURL())
      .put("ProfileBannerRetinaURL", user.getProfileBannerRetinaURL())
      .put("ProfileBannerIPadURL", user.getProfileBannerIPadURL())
      .put("ProfileBannerIPadRetinaURL", user.getProfileBannerIPadRetinaURL())
      .put("ProfileBannerMobileURL", user.getProfileBannerMobileURL())
      .put("ProfileBannerMobileRetinaURL", user.getProfileBannerMobileRetinaURL())
      .put("ProfileBackgroundTiled", user.isProfileBackgroundTiled())
      .put("Lang", user.getLang())
      .put("StatusesCount", user.getStatusesCount())
      .put("GeoEnabled", user.isGeoEnabled())
      .put("Verified", user.isVerified())
      .put("Translator", user.isTranslator())
      .put("ListedCount", user.getListedCount())
      .put("FollowRequestSent", user.isFollowRequestSent());

  List<String> withheldInCountries = new ArrayList<>();
  if (null != user.getWithheldInCountries()) {
    for (String s : user.getWithheldInCountries()) {
      withheldInCountries.add(s);
    }
  }
  struct.put("WithheldInCountries", withheldInCountries);

}
 

@Override
protected void setDateField(Struct result, String fieldName, Date value) {
  result.put(fieldName, value);
}
 

@Override
public List<SourceRecord> process() throws IOException {
  List<SourceRecord> records = new ArrayList<>(this.config.batchSize);

  while (records.size() < this.config.batchSize) {
    String[] row = this.csvReader.readNext();

    if (row == null) {
      break;
    }
    log.trace("process() - Row on line {} has {} field(s)", recordOffset(), row.length);

    Struct keyStruct = new Struct(this.config.keySchema);
    Struct valueStruct = new Struct(this.config.valueSchema);

    for (int i = 0; i < this.fieldNames.length; i++) {
      String fieldName = this.fieldNames[i];
      log.trace("process() - Processing field {}", fieldName);
      String input = row[i];
      log.trace("process() - input = '{}'", input);
      Object fieldValue = null;

      try {
        Field field = this.config.valueSchema.field(fieldName);
        if (null != field) {
          fieldValue = this.parser.parseString(field.schema(), input);
          log.trace("process() - output = '{}'", fieldValue);
          valueStruct.put(field, fieldValue);
        } else {
          log.trace("process() - Field {} is not defined in the schema.", fieldName);
        }
      } catch (Exception ex) {
        String message = String.format("Exception thrown while parsing data for '%s'. linenumber=%s", fieldName, this.recordOffset());
        throw new DataException(message, ex);
      }

      Field keyField = this.config.keySchema.field(fieldName);
      if (null != keyField) {
        log.trace("process() - Setting key field '{}' to '{}'", keyField.name(), fieldValue);
        keyStruct.put(keyField, fieldValue);
      }
    }

    if (log.isInfoEnabled() && this.csvReader.getLinesRead() % ((long) this.config.batchSize * 20) == 0) {
      log.info("Processed {} lines of {}", this.csvReader.getLinesRead(), this.inputFile);
    }

    addRecord(
        records,
        new SchemaAndValue(keyStruct.schema(), keyStruct),
        new SchemaAndValue(valueStruct.schema(), valueStruct)
    );


  }
  return records;
}
 

@Override
protected void setBytesField(Struct result, String fieldName, byte[] value) {
  result.put(fieldName, value);
}
 

@Override
protected void setStringField(Struct result, String fieldName, String value) {
  result.put(fieldName, value);
}
 

@Override
protected void setInt32Field(Struct result, String fieldName, Integer value) {
  result.put(fieldName, value);
}