Java Code Examples for org.apache.avro.generic.IndexedRecord#getSchema()

public static <T> List<IndexedRecord> makeIndexedRecords(NetSuiteClientService<?> clientService, Schema schema,
        ObjectComposer<T> objectComposer, int count) throws Exception {

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(clientService, schema, schema.getName());

    List<IndexedRecord> recordList = new ArrayList<>();

    while (count > 0) {
        T nsRecord = objectComposer.composeObject();

        IndexedRecord convertedRecord = transducer.read(nsRecord);
        Schema recordSchema = convertedRecord.getSchema();

        GenericRecord record = new GenericData.Record(recordSchema);
        for (Schema.Field field : schema.getFields()) {
            Object value = convertedRecord.get(field.pos());
            record.put(field.pos(), value);
        }

        recordList.add(record);

        count--;
    }

    return recordList;
}
 

@Test
public void testDynamicSchema(){
    Schema designSchema = SchemaBuilder.builder().record("Schema").prop(SchemaConstants.INCLUDE_ALL_FIELDS, "true").fields() //
            .name("salesforce_id").type().intType().noDefault() //
            .name("salesforce_created").type().booleanType().noDefault() //
            .endRecord();

    converter.setSchema(designSchema);

    BulkResult result = new BulkResult();
    result.setValue("Created", "true");
    result.setValue("Error", "");
    result.setValue("Id", "a0M2v00000JSnn6EAD");
    result.setValue("Success", "true");
    result.setValue("field_1__c", "vlaue 1");
    result.setValue("field_2__c", "vlaue 2");
    result.setValue("test_uk__c", "0012v00002OGo4JAAT");

    IndexedRecord indexedRecord = converter.convertToAvro(result);

    Schema runtimSchema = indexedRecord.getSchema();
    Assert.assertEquals(5,runtimSchema.getFields().size());
    Assert.assertNotNull(runtimSchema.getField("field_1__c"));
    Assert.assertNotNull(runtimSchema.getField("field_2__c"));
    Assert.assertNotNull(runtimSchema.getField("test_uk__c"));
}
 

@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    //
    successfulWrites.clear();
    rejectedWrites.clear();
    //
    inputRecord = (IndexedRecord) object;
    result.totalCount++;
    // This for dynamic which would get schema from the first record
    if (inputSchema == null) {
        inputSchema = inputRecord.getSchema();
    }
    //
    recordsToProcess.add(inputRecord);
    if (recordsToProcess.size() >= batchSize) {
        processResult(((MarketoRESTClient) client).requestCampaign(properties, recordsToProcess));
        recordsToProcess.clear();
    }
}
 

@Override
public void write(Object indexedRecordDatum) throws IOException {
    if (indexedRecordDatum == null) {
        return;
    }
    cleanWrites();
    IndexedRecord indexedRecord = (IndexedRecord) indexedRecordDatum;
    Schema indexedRecordSchema = indexedRecord.getSchema();

    int docIdFieldIndex = indexedRecordSchema.getFields().indexOf(indexedRecordSchema.getField(inputProperties.docIdColumn.getStringValue()));
    if (docIdFieldIndex == -1) {
        throw new MarkLogicException(new MarkLogicErrorCode("Can't find docId column "
                + inputProperties.docIdColumn.getStringValue() + " in input row"));
    }
    String docId = (String) indexedRecord.get(docIdFieldIndex);
    GenericData.Record matchedDocument = docContentReader.readDocument(docId);
    totalCounter++;

    documents.add(matchedDocument);
}
 

@Test
public void testManualQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    props.query.setValue("select Id from Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> outputRows = readRows(props);
    assertEquals(100, outputRows.size());
    props.module.main.schema.setValue(SchemaBuilder.builder().record("MakeRowRecord").fields()//
            .name("Id").type().nullable().stringType().noDefault() //
            .name("Name").type().nullable().stringType().noDefault() //
            .name("Owner_Name").type().nullable().stringType().noDefault() //
            .name("Owner_Id").type().nullable().stringType().noDefault().endRecord());
    props.query
            .setValue("SELECT Id, Name, Owner.Name ,Owner.Id FROM Account WHERE Name = '" + randomizedValue + "'");
    List<IndexedRecord> rowsWithForeignKey = readRows(props);

    props.module.main.schema.setValue(SchemaBuilder.builder().record("MakeRowRecord").fields()//
            .name("Id").type().nullable().stringType().noDefault() //
            .name("Name").type().nullable().stringType().noDefault() //
            .name("OwnerId").type().nullable().stringType().noDefault().endRecord());
    props.query.setValue("SELECT Id, Name, OwnerId FROM Account WHERE Name = '" + randomizedValue + "'");
    outputRows = readRows(props);

    assertEquals(rowsWithForeignKey.size(), outputRows.size());
    assertEquals(100, rowsWithForeignKey.size());
    IndexedRecord fkRecord = rowsWithForeignKey.get(0);
    IndexedRecord commonRecord = outputRows.get(0);
    assertNotNull(fkRecord);
    assertNotNull(commonRecord);
    Schema schemaFK = fkRecord.getSchema();
    Schema schemaCommon = commonRecord.getSchema();

    assertNotNull(schemaFK);
    assertNotNull(schemaCommon);
    assertEquals(commonRecord.get(schemaCommon.getField("OwnerId").pos()), fkRecord.get(schemaFK.getField("Owner_Id").pos()));

}
 

public static <T> List<IndexedRecord> makeIndexedRecords(
        NetSuiteClientService<?> clientService, Schema schema,
        ObjectComposer<T> objectComposer, int count) throws Exception {

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(clientService, schema, schema.getName());

    List<IndexedRecord> recordList = new ArrayList<>();

    while (count > 0) {
        T nsRecord = objectComposer.composeObject();

        IndexedRecord convertedRecord = transducer.read(nsRecord);
        Schema recordSchema = convertedRecord.getSchema();

        GenericRecord record = new GenericData.Record(recordSchema);
        for (Schema.Field field : schema.getFields()) {
            Object value = convertedRecord.get(field.pos());
            record.put(field.pos(), value);
        }

        recordList.add(record);

        count--;
    }

    return recordList;
}
 

@Test
public void testComplexSOQLQuery() throws Throwable {
    TSalesforceInputProperties props = createTSalesforceInputProperties(false, false);
    props.manualQuery.setValue(true);
    // Manual query with foreign key
    // Need to specify where clause to be sure that this record exists and has parent-to-child relation.
    props.query.setValue("Select Id, Name,(Select Contact.Id,Contact.Name from Account.Contacts) from Account WHERE Name = 'United Oil & Gas, UK' Limit 1");
    props.validateGuessSchema();
    List<IndexedRecord> rows = readRows(props);

    if (rows.size() > 0) {
        for (IndexedRecord row : rows) {
            Schema schema = row.getSchema();
            assertNotNull(schema.getField("Id"));
            assertNotNull(schema.getField("Name"));
            assertNotNull(schema.getField("Account_Contacts_records_Contact_Id"));
            assertNotNull(schema.getField("Account_Contacts_records_Contact_Name"));

            assertNotNull(row.get(schema.getField("Id").pos()));
            assertNotNull(row.get(schema.getField("Name").pos()));
            assertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Id").pos()));
            assertNotNull(row.get(schema.getField("Account_Contacts_records_Contact_Name").pos()));
        }
    } else {
        LOGGER.warn("Query result is empty!");
    }
}
 

@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 

public AvroRecord(final IndexedRecord record) {
    this.schema = new AvroSchema(record.getSchema());
    this.delegate = record;
    // dirty fix for Avro DateTime related logicalTypes converted to org.joda.time.DateTime
    this.delegate
            .getSchema()
            .getFields()
            .stream()
            .filter(f -> org.joda.time.DateTime.class.isInstance(this.delegate.get(f.pos())))
            .forEach(f -> this.delegate
                    .put(f.pos(), org.joda.time.DateTime.class.cast(this.delegate.get(f.pos())).getMillis()));
}
 

/**
 * Inserts resources into Jira server formed from incoming data input <br>
 * Method should be called only after {@link JiraInsertWriter#open(String)}
 * 
 * @param datum input data
 */
@Override
public void write(Object datum) throws IOException {
    if (!opened) {
        throw new IOException(MESSAGES.getMessage("error.writerNotOpened"));
    }
    result.totalCount++;
    if (datum == null) {
        return;
    }
    IndexedRecord record = getFactory(datum).convertToAvro(datum);

    if (dataSchema == null) {
        dataSchema = record.getSchema();
        Field jsonField = dataSchema.getField("json");
        if (jsonField == null) {
            throw new IOException(MESSAGES.getMessage("error.schemaNotContainJson"));
        }
        jsonPos = jsonField.pos();
    }

    String json = (String) record.get(jsonPos);
    validateRequestBody(json);

    JiraResponse response = getConnection().post(resource, json);
    handleResponse(response, json, record);
}
 

@Override
public void write(Object datum) throws IOException {
    if (!opened) {
        throw new IOException("Writer is not opened");
    }

    result.totalCount++;
    if (datum == null) {
        return;
    }

    // Data object is always IndexedRecord
    IndexedRecord record = (IndexedRecord) datum;

    Schema schema = record.getSchema();
    Schema.Field idField = schema.getField(idFieldName);
    if (idField == null) {
        throw new IOException("Schema does not contain ID field: " + idFieldName);
    }

    int idPos = idField.pos();
    Object id = record.get(idPos);
    if (id == null) {
        handleException("Record is not processed. ID is null.", new IllegalArgumentException("ID field should not be null"));
        return;
    }
    try {
        if (containsJson){
            connection.insertJsonDocument(id.toString(), createHierarchicalJson(schema, record, idPos));
        } else {
            connection.upsert(id.toString(), datum.toString());
        }
        result.successCount++;
    } catch (Exception e) {
        handleException("Record is not processed. Failed to upsert value - " + datum.toString(), e);
    }
}
 

/**
 * Checks {@link RootRecordUtils#createRootRecord(Schema, Schema)} returns Root record, which schema is Root schema
 */
@Test
public void testCreateRootRecord() {
    Schema mainSchema = SchemaBuilder.record("Main").fields() //
            .name("name").type().stringType().noDefault().endRecord(); //

    Schema outOfBandSchema = SchemaBuilder.record("OutOfBand").fields() //
            .name("id").type().intType().noDefault().endRecord(); //

    IndexedRecord rootRecord = RootRecordUtils.createRootRecord(mainSchema, outOfBandSchema);

    assertNotNull(rootRecord);
    Schema rootSchema = rootRecord.getSchema();
    assertTrue(RootSchemaUtils.isRootSchema(rootSchema));
}
 

/**
 * Checks whether input {@link IndexedRecord} is Root record
 * It checks its {@link Schema}, it should be Root schema
 * 
 * @param data data to check
 * @return true if incoming {@link IndexedRecord} is Root record; false otherwise
 */
public static boolean isRootRecord(Object data) {
    if (data == null) {
        return false;
    }
    if (!(data instanceof IndexedRecord)) {
        return false;
    }
    IndexedRecord record = (IndexedRecord) data;
    Schema recordSchema = record.getSchema();
    boolean result = RootSchemaUtils.isRootSchema(recordSchema);
    return result;
}
 

private static IndexedRecord copyValueFrom(IndexedRecord record) {
    Schema schema = record.getSchema();
    IndexedRecord result = new GenericData.Record(schema);
    List<Schema.Field> fields = schema.getFields();
    for (int i = 0; i < fields.size(); i++) {
        result.put(i, record.get(i));
    }

    return result;
}
 

public static void fetchFieldNamesAndValuesFromIndexedRecord(IndexedRecord record, List<String> fieldNames,
    List<Object> fieldValues, Schema cachedSchema) {
  // Please note that record schema and cached schema could be different due to schema evolution.
  // Always represent record schema in the form of cached schema. This approach has the side-effect
  // of dropping the newly added fields in the scenarios where the record schema has newer version
  // than the cached schema. [TODO: SAMZA-1679]
  Schema recordSchema = record.getSchema();
  fieldNames.addAll(cachedSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList()));
  fieldValues.addAll(fieldNames.stream()
      .map(f -> convertToJavaObject(
          recordSchema.getField(f) != null ? record.get(recordSchema.getField(f).pos()) : null,
          getNonNullUnionSchema(cachedSchema.getField(f).schema()))) // get schema from cachedSchema
      .collect(Collectors.toList()));
}
 

private byte[] encodeRecord(IndexedRecord record) throws Exception {
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = record.getSchema();
  EncoderFactory encoderfactory = new EncoderFactory();
  BinaryEncoder encoder = encoderfactory.binaryEncoder(out, null);
  DatumWriter<IndexedRecord> writer;
  if (record instanceof SpecificRecord) {
    writer = new SpecificDatumWriter<>(schema);
  } else {
    writer = new GenericDatumWriter<>(schema);
  }
  writer.write(record, encoder);
  encoder.flush(); //encoder may buffer
  return out.toByteArray();
}
 

@Test
public void testIncludeAllFields() throws Exception {
    NetSuiteClientService<?> connection = webServiceTestFixture.getClientService();

    connection.login();

    TypeDesc basicTypeDesc = connection.getBasicMetaData().getTypeInfo("Opportunity");
    Schema schema = getDynamicSchema();

    NsObjectInputTransducer transducer = new NsObjectInputTransducer(connection, schema, basicTypeDesc.getTypeName());

    SearchResultSet<Record> rs = connection.newSearch()
            .target(basicTypeDesc.getTypeName())
            .search();

    TypeDesc typeDesc = connection.getMetaDataSource().getTypeInfo(basicTypeDesc.getTypeName());

    int count = 0;
    while (count++ < connection.getSearchPageSize() && rs.next()) {
        Record record = rs.get();
        IndexedRecord indexedRecord = transducer.read(record);
        logger.debug("Indexed record: {}", indexedRecord);

        Schema recordSchema = indexedRecord.getSchema();
        assertEquals(typeDesc.getFields().size(), recordSchema.getFields().size());

        for (FieldDesc fieldDesc : typeDesc.getFields()) {
            String fieldName = fieldDesc.getName();
            Schema.Field field = recordSchema.getField(fieldName);
            assertNotNull(field);

            Object value = indexedRecord.get(field.pos());
        }
    }
    if (count == 0) {
        throw new IllegalStateException("No records");
    }
}
 

private Schema inferSchemaRecord(IndexedRecord in) {
    return in.getSchema();
}
 

@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    cleanWrites();
    //
    inputRecord = (IndexedRecord) object;
    result.totalCount++;
    // This for dynamic which would get schema from the first record
    if (inputSchema == null) {
        inputSchema = inputRecord.getSchema();
        if (isDynamic) {
            adaptSchemaToDynamic();
        }
    }
    // switch between column name in design and column value for runtime
    properties.leadKeyValues.setValue(String.valueOf(inputRecord.get(inputSchema.getField(leadKeyColumn).pos())));
    //
    for (int i = 0; i < getRetryAttemps(); i++) {
        result.apiCalls++;
        MarketoRecordResult mktoResult = client.getMultipleLeads(properties, null);
        //
        if (!mktoResult.isSuccess()) {
            if (dieOnError) {
                throw new IOException(mktoResult.getErrorsString());
            }
            // is recoverable error
            if (client.isErrorRecoverable(mktoResult.getErrors())) {
                LOG.debug("Recoverable error during operation : `{}`. Retrying...", mktoResult.getErrorsString());
                waitForRetryAttempInterval();
                continue;
            } else {
                LOG.error("Unrecoverable error : `{}`.", mktoResult.getErrorsString());
                break;
            }
        } else {
            for (IndexedRecord record : mktoResult.getRecords()) {
                if (record != null) {
                    this.result.successCount++;
                    successfulWrites.add(record);
                }
            }
            break;
        }
    }
}
 

public void write(Object datum) throws IOException {
    result.totalCount++;

    cleanWrites();
    
    IndexedRecord inputRecord = this.getGenericIndexedRecordConverter(datum).convertToAvro(datum);

    Schema inputSchema = inputRecord.getSchema();

    try {

        sink.fillParameters(cs, componentSchema, inputSchema, inputRecord, setting);

        cs.execute();

        if (indexedRecordCreator == null) {
            indexedRecordCreator = new JDBCSPIndexedRecordCreator();
            indexedRecordCreator.init(componentSchema, outputSchema, setting);
        }

        IndexedRecord outputRecord = indexedRecordCreator.createOutputIndexedRecord(cs, inputRecord);

        successfulWrites.add(outputRecord);
    } catch (Exception e) {
        throw CommonUtils.newComponentException(e);
    }

}