io.confluent.kafka.schemaregistry.client.SchemaMetadata Java Examples

@Test
void shouldHaveLatestSchemaVersion() throws IOException, RestClientException {
    final Schema valueSchema1 = createSchema("value_schema");
    final String topic = "test-topic";
    final int id1 = this.schemaRegistry.registerValueSchema(topic, valueSchema1);

    final List<Schema.Field> fields = Collections.singletonList(
            new Schema.Field("f1", Schema.create(Schema.Type.STRING), "", (Object) null));
    final Schema valueSchema2 = Schema.createRecord("value_schema", "no doc", "", false, fields);
    final int id2 = this.schemaRegistry.registerValueSchema(topic, valueSchema2);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isEqualTo(2);

    final SchemaMetadata metadata = this.schemaRegistry.getSchemaRegistryClient().getLatestSchemaMetadata(topic + "-value");
    final int metadataId = metadata.getId();
    assertThat(metadataId).isNotEqualTo(id1);
    assertThat(metadataId).isEqualTo(id2);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema2);
}
 

@Test
void shouldNotHaveSchemaVersionsForDeletedSubject() throws IOException, RestClientException {
    final Schema valueSchema = createSchema("value_schema");
    final String topic = "test-topic";
    final int id = this.schemaRegistry.registerValueSchema(topic, valueSchema);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isOne();

    final SchemaMetadata metadata = this.schemaRegistry.getSchemaRegistryClient().getSchemaMetadata(topic + "-value", versions.get(0));
    assertThat(metadata.getId()).isEqualTo(id);
    assertThat(this.schemaRegistry.getSchemaRegistryClient().getLatestSchemaMetadata(topic + "-value"))
            .isNotNull();
    this.schemaRegistry.deleteValueSchema(topic);
    assertThatExceptionOfType(RestClientException.class)
            .isThrownBy(() -> this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value"))
            .satisfies(e -> assertThat(e.getStatus()).isEqualTo(HTTP_NOT_FOUND));
    assertThatExceptionOfType(RestClientException.class)
            .isThrownBy(() -> this.schemaRegistry.getSchemaRegistryClient().getSchemaMetadata(topic + "-value", versions.get(0)))
            .satisfies(e -> assertThat(e.getStatus()).isEqualTo(HTTP_NOT_FOUND));
    assertThatExceptionOfType(RestClientException.class)
            .isThrownBy(() -> this.schemaRegistry.getSchemaRegistryClient().getLatestSchemaMetadata(topic + "-value"))
            .satisfies(e -> assertThat(e.getStatus()).isEqualTo(HTTP_NOT_FOUND));
}
 

@Test
void shouldHaveSchemaVersions() throws IOException, RestClientException {
    final Schema valueSchema = this.createSchema("value_schema");
    final String topic = "test-topic";
    final int id = this.schemaRegistry.registerValueSchema(topic, valueSchema);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isOne();

    final SchemaMetadata metadata =
            this.schemaRegistry.getSchemaRegistryClient().getSchemaMetadata(topic + "-value", versions.get(0));
    assertThat(metadata.getId()).isEqualTo(id);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema);
}
 

@Test
void shouldHaveLatestSchemaVersion() throws IOException, RestClientException {
    final Schema valueSchema1 = this.createSchema("value_schema");
    final String topic = "test-topic";
    final int id1 = this.schemaRegistry.registerValueSchema(topic, valueSchema1);

    final List<Schema.Field> fields = Collections.singletonList(
            new Schema.Field("f1", Schema.create(Schema.Type.STRING), "", (Object) null));
    final Schema valueSchema2 = Schema.createRecord("value_schema", "no doc", "", false, fields);
    final int id2 = this.schemaRegistry.registerValueSchema(topic, valueSchema2);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isEqualTo(2);

    final SchemaMetadata metadata =
            this.schemaRegistry.getSchemaRegistryClient().getLatestSchemaMetadata(topic + "-value");
    final int metadataId = metadata.getId();
    assertThat(metadataId).isNotEqualTo(id1);
    assertThat(metadataId).isEqualTo(id2);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema2);
}
 

@Test
public void shouldHaveSchemaVersions() throws IOException, RestClientException {
    final Schema valueSchema = this.createSchema("value_schema");
    final String topic = "test-topic";
    final int id = this.schemaRegistry.registerValueSchema(topic, valueSchema);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isOne();

    final SchemaMetadata metadata =
            this.schemaRegistry.getSchemaRegistryClient().getSchemaMetadata(topic + "-value", versions.get(0));
    assertThat(metadata.getId()).isEqualTo(id);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema);
}
 

@Test
public void shouldHaveLatestSchemaVersion() throws IOException, RestClientException {
    final Schema valueSchema1 = this.createSchema("value_schema");
    final String topic = "test-topic";
    final int id1 = this.schemaRegistry.registerValueSchema(topic, valueSchema1);

    final List<Schema.Field> fields = Collections.singletonList(
            new Schema.Field("f1", Schema.create(Schema.Type.STRING), "", (Object) null));
    final Schema valueSchema2 = Schema.createRecord("value_schema", "no doc", "", false, fields);
    final int id2 = this.schemaRegistry.registerValueSchema(topic, valueSchema2);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isEqualTo(2);

    final SchemaMetadata metadata =
            this.schemaRegistry.getSchemaRegistryClient().getLatestSchemaMetadata(topic + "-value");
    final int metadataId = metadata.getId();
    assertThat(metadataId).isNotEqualTo(id1);
    assertThat(metadataId).isEqualTo(id2);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema2);
}
 

@Test
void getAndUpdateMetadataOfSchema() throws IOException, RestClientException, TimeoutException {
    Schema schema = new Schema.Parser().parse("{\"type\":\"record\",\"name\":\"myrecord1\",\"fields\":[{\"name\":\"foo\",\"type\":\"string\"}]}");
    String schemaSubject = TestUtils.generateArtifactId();

    int schemaId = createArtifactViaConfluentClient(schema, schemaSubject);

    schema = confluentService.getById(schemaId);
    SchemaMetadata schemaMetadata = confluentService.getSchemaMetadata(schemaSubject, 1);

    LOGGER.info("Scheme name: {} has following metadata: {}", schema.getFullName(), schemaMetadata.getSchema());

    assertThat(schemaMetadata.getId(), is(schemaId));
    assertThat(schemaMetadata.getVersion(), is(1));
    assertThat("{\"type\":\"record\",\"name\":\"myrecord1\",\"fields\":[{\"name\":\"foo\",\"type\":\"string\"}]}", is(schemaMetadata.getSchema()));
    // IMPORTANT NOTE: we can not test schema metadata, because they are mapping on the same endpoint when we are creating the schema...
}
 

@Test
public void shouldNotPassAvroCheckIfSchemaDoesNotExist() throws Exception {
  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  SchemaMetadata schemaMetadata = new SchemaMetadata(1, 1, null);
  expect(schemaRegistryClient.getLatestSchemaMetadata(anyString())).andReturn(schemaMetadata);
  replay(schemaRegistryClient);
  AbstractStreamCreateStatement abstractStreamCreateStatement = getAbstractStreamCreateStatement
      ("CREATE STREAM S1 WITH "
       + "(kafka_topic='s1_topic', "
       + "value_format='avro' );");
  try {
    avroUtil.checkAndSetAvroSchema(abstractStreamCreateStatement, new HashMap<>(), schemaRegistryClient);
    fail();
  } catch (Exception e) {
    assertThat("Expected different message message.", e.getMessage().trim(),
        equalTo("Unable to verify the AVRO schema is compatible with KSQL. null"));
  }
}
 

@Test
public void shouldPassAvroCheck() throws Exception {
  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  SchemaMetadata schemaMetadata = new SchemaMetadata(1, 1, ordersAveroSchemaStr);
  expect(schemaRegistryClient.getLatestSchemaMetadata(anyString())).andReturn(schemaMetadata);
  replay(schemaRegistryClient);
  AbstractStreamCreateStatement abstractStreamCreateStatement = getAbstractStreamCreateStatement
      ("CREATE STREAM S1 WITH "
                                                                 + "(kafka_topic='s1_topic', "
                                   + "value_format='avro' );");
  Pair<AbstractStreamCreateStatement, String> checkResult = avroUtil.checkAndSetAvroSchema(abstractStreamCreateStatement, new HashMap<>(), schemaRegistryClient);
  AbstractStreamCreateStatement newAbstractStreamCreateStatement = checkResult.getLeft();
  assertThat(newAbstractStreamCreateStatement.getElements(), equalTo(Arrays.asList(
      new TableElement("ORDERTIME", "BIGINT"),
      new TableElement("ORDERID", "BIGINT"),
      new TableElement("ITEMID", "VARCHAR"),
      new TableElement("ORDERUNITS", "DOUBLE"),
      new TableElement("ARRAYCOL", "ARRAY<DOUBLE>"),
      new TableElement("MAPCOL", "MAP<VARCHAR,DOUBLE>")
      )));
}
 

@Override
public ResponseDefinition transform(final Request request, final ResponseDefinition responseDefinition,
                                    final FileSource files, final Parameters parameters) {
    String versionStr = Iterables.get(this.urlSplitter.split(request.getUrl()), 3);
    SchemaMetadata metadata;
    if (versionStr.equals("latest")) {
        metadata = SchemaRegistryMock.this.getSubjectVersion(getSubject(request), versionStr);
    } else {
        int version = Integer.parseInt(versionStr);
        metadata = SchemaRegistryMock.this.getSubjectVersion(getSubject(request), version);
    }
    return ResponseDefinitionBuilder.jsonResponse(metadata);
}
 

/**
 * Looks up schema id for the specified subject from the schema registry
 * @param subject subject for which schema Id must be looked up.
 * @return the schema id
 * @throws SchemaRegistryException if there is an error with the registry client
 */
public int getSchemaIdFromSubject(String subject) throws SchemaRegistryException {
  try {
    SchemaMetadata metadata = registryClient.getLatestSchemaMetadata(subject);
    return metadata.getId();
  } catch (IOException | RestClientException e) {
    throw new SchemaRegistryException(e);
  }
}
 

/**
 * Loads and parses a schema for the specified subject from the schema registry
 * @param subject subject for which to fetch the latest version of a schema.
 * @return parsed avro schema
 * @throws SchemaRegistryException if there is an error with the registry client
 */
public Schema loadFromRegistry(String subject) throws SchemaRegistryException {
  try {
    SchemaMetadata metadata = registryClient.getLatestSchemaMetadata(subject);
    return registryClient.getByID(metadata.getId());
  } catch (IOException | RestClientException e) {
    throw new SchemaRegistryException(e);
  }
}
 

private SchemaMetadata getSchemaMetadata() {
  try {
    return (version == null)
        ? getSchemaRegistryClient().getLatestSchemaMetadata(subject)
        : getSchemaRegistryClient().getSchemaMetadata(subject, version);
  } catch (IOException | RestClientException e) {
    throw new RuntimeException("Unable to get latest schema metadata for subject: " + subject, e);
  }
}
 

public synchronized SchemaMetadata getLatestSchemaMetadata(String subject) throws IOException, RestClientException {
    io.confluent.kafka.schemaregistry.client.rest.entities.Schema response = this.restService.getLatestVersion(subject);
    int id = response.getId().intValue();
    int version = response.getVersion().intValue();
    String schema = response.getSchema();
    return new SchemaMetadata(id, version, schema);
}
 

private SchemaMetadata fetchSchemaMetadata(
    AbstractStreamCreateStatement abstractStreamCreateStatement,
    SchemaRegistryClient schemaRegistryClient,
    String kafkaTopicName
) throws IOException, RestClientException {

  if (abstractStreamCreateStatement.getProperties().containsKey(KsqlConstants.AVRO_SCHEMA_ID)) {
    int schemaId;
    try {
      schemaId = Integer.parseInt(
          StringUtil.cleanQuotes(
              abstractStreamCreateStatement
                  .getProperties()
                  .get(KsqlConstants.AVRO_SCHEMA_ID)
                  .toString()
          )
      );
    } catch (NumberFormatException e) {
      throw new KsqlException(String.format(
          "Invalid schema id property: %s.",
          abstractStreamCreateStatement
              .getProperties()
              .get(KsqlConstants.AVRO_SCHEMA_ID)
              .toString()
      ));
    }
    return schemaRegistryClient.getSchemaMetadata(
        kafkaTopicName + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX,
        schemaId
    );
  } else {
    return schemaRegistryClient.getLatestSchemaMetadata(
        kafkaTopicName + KsqlConstants.SCHEMA_REGISTRY_VALUE_SUFFIX
    );
  }
}
 

private SchemaMetadata getSubjectVersion(String subject, Object version) {
    log.debug("Requesting version {} for subject {}", version, subject);
    try {
        if (version instanceof String && version.equals("latest")) {
            return this.schemaRegistryClient.getLatestSchemaMetadata(subject);
        } else if (version instanceof Number) {
            return this.schemaRegistryClient.getSchemaMetadata(subject, ((Number) version).intValue());
        } else {
            throw new IllegalArgumentException("Only 'latest' or integer versions are allowed");
        }
    } catch (IOException | RestClientException e) {
        throw new IllegalStateException("Internal error in mock schema registry client", e);
    }
}
 

@Test
void shouldHaveSchemaVersions() throws IOException, RestClientException {
    final Schema valueSchema = createSchema("value_schema");
    final String topic = "test-topic";
    final int id = this.schemaRegistry.registerValueSchema(topic, valueSchema);

    final List<Integer> versions = this.schemaRegistry.getSchemaRegistryClient().getAllVersions(topic + "-value");
    assertThat(versions.size()).isOne();

    final SchemaMetadata metadata = this.schemaRegistry.getSchemaRegistryClient().getSchemaMetadata(topic + "-value", versions.get(0));
    assertThat(metadata.getId()).isEqualTo(id);
    final String schemaString = metadata.getSchema();
    final Schema retrievedSchema = new Schema.Parser().parse(schemaString);
    assertThat(retrievedSchema).isEqualTo(valueSchema);
}
 

@Override
public ResponseDefinition transform(final Request request, final ResponseDefinition responseDefinition,
        final FileSource files, final Parameters parameters) {
    final String versionStr = Iterables.get(this.urlSplitter.split(request.getUrl()), 3);
    final SchemaMetadata metadata;
    if (versionStr.equals("latest")) {
        metadata = SchemaRegistryMock.this.getSubjectVersion(this.getSubject(request), versionStr);
    } else {
        final int version = Integer.parseInt(versionStr);
        metadata = SchemaRegistryMock.this.getSubjectVersion(this.getSubject(request), version);
    }
    return ResponseDefinitionBuilder.jsonResponse(metadata);
}
 

private SchemaMetadata getSubjectVersion(final String subject, final Object version) {
    log.debug("Requesting version {} for subject {}", version, subject);
    try {
        if (version instanceof String && version.equals("latest")) {
            return this.schemaRegistryClient.getLatestSchemaMetadata(subject);
        } else if (version instanceof Number) {
            return this.schemaRegistryClient.getSchemaMetadata(subject, ((Number) version).intValue());
        } else {
            throw new IllegalArgumentException("Only 'latest' or integer versions are allowed");
        }
    } catch (final IOException | RestClientException e) {
        throw new IllegalStateException("Internal error in mock schema registry client", e);
    }
}
 

public Pair<AbstractStreamCreateStatement, String> checkAndSetAvroSchema(
    final AbstractStreamCreateStatement abstractStreamCreateStatement,
    final Map<String, Object> streamsProperties,
    final SchemaRegistryClient schemaRegistryClient
) {

  Map<String, Expression> ddlProperties = abstractStreamCreateStatement.getProperties();
  if (!ddlProperties.containsKey(DdlConfig.VALUE_FORMAT_PROPERTY)) {
    throw new KsqlException(String.format(
        "%s should be set in WITH clause of CREATE STREAM/TABLE statement.",
        DdlConfig.VALUE_FORMAT_PROPERTY
    )
    );
  }
  final String serde = StringUtil.cleanQuotes(
      ddlProperties.get(DdlConfig.VALUE_FORMAT_PROPERTY).toString()
  );
  if (!serde.equalsIgnoreCase(DataSource.AVRO_SERDE_NAME)) {
    return new Pair<>(abstractStreamCreateStatement, null);
  }

  String kafkaTopicName = StringUtil.cleanQuotes(
      ddlProperties.get(DdlConfig.KAFKA_TOPIC_NAME_PROPERTY).toString()
  );
  try {
    // If the schema is not specified infer it from the Avro schema in Schema Registry.
    if (abstractStreamCreateStatement.getElements().isEmpty()) {
      SchemaMetadata schemaMetadata = fetchSchemaMetadata(
          abstractStreamCreateStatement,
          schemaRegistryClient,
          kafkaTopicName
      );

      String avroSchemaString = schemaMetadata.getSchema();
      streamsProperties.put(DdlConfig.AVRO_SCHEMA, avroSchemaString);
      Schema schema = SerDeUtil.getSchemaFromAvro(avroSchemaString);
      AbstractStreamCreateStatement abstractStreamCreateStatementCopy = addAvroFields(
          abstractStreamCreateStatement,
          schema,
          schemaMetadata.getId()
      );
      return new Pair<>(
          abstractStreamCreateStatementCopy,
          SqlFormatter.formatSql(abstractStreamCreateStatementCopy)
      );
    } else {
      return new Pair<>(abstractStreamCreateStatement, null);
    }
  } catch (Exception e) {
    String errorMessage = String.format(
        " Unable to verify the AVRO schema is compatible with KSQL. %s ",
        e.getMessage()
    );
    throw new KsqlException(errorMessage);
  }
}
 

@Override
public SchemaMetadata getSchemaMetadata(final String s, final int i) throws IOException, RestClientException
{
  return null;
}
 

@Override
public SchemaMetadata getLatestSchemaMetadata(final String s) throws IOException, RestClientException
{
  return null;
}
 

public SchemaMetadata getSchemaMetadata(String subject, int version) throws IOException, RestClientException {
    io.confluent.kafka.schemaregistry.client.rest.entities.Schema response = this.restService.getVersion(subject, version);
    int id = response.getId().intValue();
    String schema = response.getSchema();
    return new SchemaMetadata(id, version, schema);
}